HMVMXR0.cpp@ 80099

Last change on this file since 80099 was 80099, checked in by vboxsync, 6 years ago
VMM: Kicking out 32-bit host support - VMX [build fix]. bugref:9511
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 706.7 KB

Line
1	/* $Id: HMVMXR0.cpp 80099 2019-08-01 05:39:51Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/gim.h>
38	#include <VBox/vmm/apic.h>
39	#ifdef VBOX_WITH_REM
40	# include <VBox/vmm/rem.h>
41	#endif
42	#include "HMInternal.h"
43	#include <VBox/vmm/vm.h>
44	#include <VBox/vmm/hmvmxinline.h>
45	#include "HMVMXR0.h"
46	#include "dtrace/VBoxVMM.h"
47
48	#ifdef DEBUG_ramshankar
49	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
50	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
51	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
52	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
53	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
54	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
55	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
56	# define HMVMX_ALWAYS_TRAP_PF
57	# define HMVMX_ALWAYS_FLUSH_TLB
58	# define HMVMX_ALWAYS_SWAP_EFER
59	#endif
60
61
62	/*********************************************************************************************************************************
63	* Defined Constants And Macros *
64	*********************************************************************************************************************************/
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/**
75	* Flags to skip redundant reads of some common VMCS fields that are not part of
76	* the guest-CPU or VCPU state but are needed while handling VM-exits.
77	*/
78	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
79	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
80	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
81	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
82	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
83	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
84	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
85	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
86	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
87
88	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
89	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
90	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
91	\| HMVMX_READ_EXIT_INSTR_LEN \
92	\| HMVMX_READ_IDT_VECTORING_INFO \
93	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
94
95	/** Assert that all the given fields have been read from the VMCS. */
96	#ifdef VBOX_STRICT
97	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
98	do { \
99	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
100	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); \
101	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
102	} while (0)
103	#else
104	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
105	#endif
106
107	/**
108	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
109	* guest using hardware-assisted VMX.
110	*
111	* This excludes state like GPRs (other than RSP) which are always are
112	* swapped and restored across the world-switch and also registers like EFER,
113	* MSR which cannot be modified by the guest without causing a VM-exit.
114	*/
115	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
116	\| CPUMCTX_EXTRN_RFLAGS \
117	\| CPUMCTX_EXTRN_RSP \
118	\| CPUMCTX_EXTRN_SREG_MASK \
119	\| CPUMCTX_EXTRN_TABLE_MASK \
120	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
121	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
122	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
123	\| CPUMCTX_EXTRN_TSC_AUX \
124	\| CPUMCTX_EXTRN_OTHER_MSRS \
125	\| CPUMCTX_EXTRN_CR0 \
126	\| CPUMCTX_EXTRN_CR3 \
127	\| CPUMCTX_EXTRN_CR4 \
128	\| CPUMCTX_EXTRN_DR7 \
129	\| CPUMCTX_EXTRN_HWVIRT \
130	\| CPUMCTX_EXTRN_HM_VMX_MASK)
131
132	/**
133	* Exception bitmap mask for real-mode guests (real-on-v86).
134	*
135	* We need to intercept all exceptions manually except:
136	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
137	* due to bugs in Intel CPUs.
138	* - \#PF need not be intercepted even in real-mode if we have nested paging
139	* support.
140	*/
141	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
142	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
143	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
144	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
145	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
146	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
147	\| RT_BIT(X86_XCPT_XF))
148
149	/** Maximum VM-instruction error number. */
150	#define HMVMX_INSTR_ERROR_MAX 28
151
152	/** Profiling macro. */
153	#ifdef HM_PROFILE_EXIT_DISPATCH
154	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
156	#else
157	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
158	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
159	#endif
160
161	/** Assert that preemption is disabled or covered by thread-context hooks. */
162	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
163	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
164
165	/** Assert that we haven't migrated CPUs when thread-context hooks are not
166	* used. */
167	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
168	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
169	("Illegal migration! Entered on CPU %u Current %u\n", \
170	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
171
172	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
173	* context. */
174	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
175	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
176	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
177
178	/** Log the VM-exit reason with an easily visible marker to identify it in a
179	* potential sea of logging data. */
180	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
181	do { \
182	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
183	HMGetVmxExitName(a_uExitReason))); \
184	} while (0) \
185
186
187	/*********************************************************************************************************************************
188	* Structures and Typedefs *
189	*********************************************************************************************************************************/
190	/**
191	* VMX per-VCPU transient state.
192	*
193	* A state structure for holding miscellaneous information across
194	* VMX non-root operation and restored after the transition.
195	*/
196	typedef struct VMXTRANSIENT
197	{
198	/** The host's rflags/eflags. */
199	RTCCUINTREG fEFlags;
200
201	/** The guest's TPR value used for TPR shadowing. */
202	uint8_t u8GuestTpr;
203	/** Alignment. */
204	uint8_t abAlignment0[7];
205
206	/** The basic VM-exit reason. */
207	uint16_t uExitReason;
208	/** Alignment. */
209	uint16_t u16Alignment0;
210	/** The VM-exit interruption error code. */
211	uint32_t uExitIntErrorCode;
212	/** The VM-exit exit code qualification. */
213	uint64_t uExitQual;
214	/** The Guest-linear address. */
215	uint64_t uGuestLinearAddr;
216	/** The Guest-physical address. */
217	uint64_t uGuestPhysicalAddr;
218
219	/** The VM-exit interruption-information field. */
220	uint32_t uExitIntInfo;
221	/** The VM-exit instruction-length field. */
222	uint32_t cbInstr;
223	/** The VM-exit instruction-information field. */
224	VMXEXITINSTRINFO ExitInstrInfo;
225	/** Whether the VM-entry failed or not. */
226	bool fVMEntryFailed;
227	/** Whether we are currently executing a nested-guest. */
228	bool fIsNestedGuest;
229	/** Alignment. */
230	uint8_t abAlignment1[2];
231
232	/** The VM-entry interruption-information field. */
233	uint32_t uEntryIntInfo;
234	/** The VM-entry exception error code field. */
235	uint32_t uEntryXcptErrorCode;
236	/** The VM-entry instruction length field. */
237	uint32_t cbEntryInstr;
238
239	/** IDT-vectoring information field. */
240	uint32_t uIdtVectoringInfo;
241	/** IDT-vectoring error code. */
242	uint32_t uIdtVectoringErrorCode;
243
244	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
245	uint32_t fVmcsFieldsRead;
246
247	/** Whether the guest debug state was active at the time of VM-exit. */
248	bool fWasGuestDebugStateActive;
249	/** Whether the hyper debug state was active at the time of VM-exit. */
250	bool fWasHyperDebugStateActive;
251	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
252	bool fUpdatedTscOffsettingAndPreemptTimer;
253	/** Whether the VM-exit was caused by a page-fault during delivery of a
254	* contributory exception or a page-fault. */
255	bool fVectoringDoublePF;
256	/** Whether the VM-exit was caused by a page-fault during delivery of an
257	* external interrupt or NMI. */
258	bool fVectoringPF;
259	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
260	* area after VM-exit. */
261	bool fRemoveTscAuxMsr;
262	bool afAlignment0[2];
263
264	/** The VMCS info. object. */
265	PVMXVMCSINFO pVmcsInfo;
266	} VMXTRANSIENT;
267	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
268	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
269	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
271	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
272	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
273	/** Pointer to VMX transient state. */
274	typedef VMXTRANSIENT *PVMXTRANSIENT;
275	/** Pointer to a const VMX transient state. */
276	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
277
278	/**
279	* Memory operand read or write access.
280	*/
281	typedef enum VMXMEMACCESS
282	{
283	VMXMEMACCESS_READ = 0,
284	VMXMEMACCESS_WRITE = 1
285	} VMXMEMACCESS;
286
287	/**
288	* VMX VM-exit handler.
289	*
290	* @returns Strict VBox status code (i.e. informational status codes too).
291	* @param pVCpu The cross context virtual CPU structure.
292	* @param pVmxTransient The VMX-transient structure.
293	*/
294	#ifndef HMVMX_USE_FUNCTION_TABLE
295	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
296	#else
297	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
298	/** Pointer to VM-exit handler. */
299	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
300	#endif
301
302	/**
303	* VMX VM-exit handler, non-strict status code.
304	*
305	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
306	*
307	* @returns VBox status code, no informational status code returned.
308	* @param pVCpu The cross context virtual CPU structure.
309	* @param pVmxTransient The VMX-transient structure.
310	*
311	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
312	* use of that status code will be replaced with VINF_EM_SOMETHING
313	* later when switching over to IEM.
314	*/
315	#ifndef HMVMX_USE_FUNCTION_TABLE
316	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
317	#else
318	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
319	#endif
320
321
322	/*********************************************************************************************************************************
323	* Internal Functions *
324	*********************************************************************************************************************************/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
327	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
328	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
329	#else
330	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
331	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
332	#endif
333	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
334	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
335	#endif
336
337	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
338
339	/** @name VM-exit handler prototypes.
340	* @{
341	*/
342	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
343	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
344	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
345	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
346	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
347	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
348	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
349	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
350	static FNVMXEXITHANDLER hmR0VmxExitHlt;
351	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
352	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
353	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
354	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
355	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
356	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
357	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
358	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
359	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
360	static FNVMXEXITHANDLER hmR0VmxExitVmread;
361	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
362	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
363	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
364	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
365	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
366	#endif
367	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
368	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
369	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
370	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
371	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
372	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
373	static FNVMXEXITHANDLER hmR0VmxExitMwait;
374	static FNVMXEXITHANDLER hmR0VmxExitMtf;
375	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
376	static FNVMXEXITHANDLER hmR0VmxExitPause;
377	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
378	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
379	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
380	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
381	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
382	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
383	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
384	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
385	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
386	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
389	/** @} */
390
391	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
392	/** @name Nested-guest VM-exit handler prototypes.
393	* @{
394	*/
395	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
396	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
399	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
400	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
401	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
402	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
403	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
404	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
405	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
406	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
407	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
408	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
409	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
410	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
411	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
412	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
413	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
415	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
416	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
417	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
418	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
419	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
420	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
422	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
423	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
424	/** @} */
425	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
426
427
428	/*********************************************************************************************************************************
429	* Global Variables *
430	*********************************************************************************************************************************/
431	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
432	/**
433	* Array of all VMCS fields.
434	* Any fields added to the VT-x spec. should be added here.
435	*
436	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
437	* of nested-guests.
438	*/
439	static const uint32_t g_aVmcsFields[] =
440	{
441	/* 16-bit control fields. */
442	VMX_VMCS16_VPID,
443	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
444	VMX_VMCS16_EPTP_INDEX,
445
446	/* 16-bit guest-state fields. */
447	VMX_VMCS16_GUEST_ES_SEL,
448	VMX_VMCS16_GUEST_CS_SEL,
449	VMX_VMCS16_GUEST_SS_SEL,
450	VMX_VMCS16_GUEST_DS_SEL,
451	VMX_VMCS16_GUEST_FS_SEL,
452	VMX_VMCS16_GUEST_GS_SEL,
453	VMX_VMCS16_GUEST_LDTR_SEL,
454	VMX_VMCS16_GUEST_TR_SEL,
455	VMX_VMCS16_GUEST_INTR_STATUS,
456	VMX_VMCS16_GUEST_PML_INDEX,
457
458	/* 16-bits host-state fields. */
459	VMX_VMCS16_HOST_ES_SEL,
460	VMX_VMCS16_HOST_CS_SEL,
461	VMX_VMCS16_HOST_SS_SEL,
462	VMX_VMCS16_HOST_DS_SEL,
463	VMX_VMCS16_HOST_FS_SEL,
464	VMX_VMCS16_HOST_GS_SEL,
465	VMX_VMCS16_HOST_TR_SEL,
466
467	/* 64-bit control fields. */
468	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
469	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
470	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
471	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
472	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
473	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
474	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
475	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
476	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
477	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
478	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
479	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
480	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
481	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
482	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
483	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
484	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
485	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
486	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
487	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
488	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
489	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
490	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
491	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
492	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
493	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
494	VMX_VMCS64_CTRL_EPTP_FULL,
495	VMX_VMCS64_CTRL_EPTP_HIGH,
496	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
497	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
498	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
499	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
500	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
501	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
502	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
503	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
504	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
505	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
506	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
507	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
508	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
509	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
510	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
511	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
512	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
513	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
514	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
515	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
516	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
517	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
518
519	/* 64-bit read-only data fields. */
520	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
521	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
522
523	/* 64-bit guest-state fields. */
524	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
525	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
526	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
527	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
528	VMX_VMCS64_GUEST_PAT_FULL,
529	VMX_VMCS64_GUEST_PAT_HIGH,
530	VMX_VMCS64_GUEST_EFER_FULL,
531	VMX_VMCS64_GUEST_EFER_HIGH,
532	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
533	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
534	VMX_VMCS64_GUEST_PDPTE0_FULL,
535	VMX_VMCS64_GUEST_PDPTE0_HIGH,
536	VMX_VMCS64_GUEST_PDPTE1_FULL,
537	VMX_VMCS64_GUEST_PDPTE1_HIGH,
538	VMX_VMCS64_GUEST_PDPTE2_FULL,
539	VMX_VMCS64_GUEST_PDPTE2_HIGH,
540	VMX_VMCS64_GUEST_PDPTE3_FULL,
541	VMX_VMCS64_GUEST_PDPTE3_HIGH,
542	VMX_VMCS64_GUEST_BNDCFGS_FULL,
543	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
544
545	/* 64-bit host-state fields. */
546	VMX_VMCS64_HOST_PAT_FULL,
547	VMX_VMCS64_HOST_PAT_HIGH,
548	VMX_VMCS64_HOST_EFER_FULL,
549	VMX_VMCS64_HOST_EFER_HIGH,
550	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
551	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
552
553	/* 32-bit control fields. */
554	VMX_VMCS32_CTRL_PIN_EXEC,
555	VMX_VMCS32_CTRL_PROC_EXEC,
556	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
557	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
558	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
559	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
560	VMX_VMCS32_CTRL_EXIT,
561	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
562	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
563	VMX_VMCS32_CTRL_ENTRY,
564	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
565	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
566	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
567	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
568	VMX_VMCS32_CTRL_TPR_THRESHOLD,
569	VMX_VMCS32_CTRL_PROC_EXEC2,
570	VMX_VMCS32_CTRL_PLE_GAP,
571	VMX_VMCS32_CTRL_PLE_WINDOW,
572
573	/* 32-bits read-only fields. */
574	VMX_VMCS32_RO_VM_INSTR_ERROR,
575	VMX_VMCS32_RO_EXIT_REASON,
576	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
577	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
578	VMX_VMCS32_RO_IDT_VECTORING_INFO,
579	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
580	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
581	VMX_VMCS32_RO_EXIT_INSTR_INFO,
582
583	/* 32-bit guest-state fields. */
584	VMX_VMCS32_GUEST_ES_LIMIT,
585	VMX_VMCS32_GUEST_CS_LIMIT,
586	VMX_VMCS32_GUEST_SS_LIMIT,
587	VMX_VMCS32_GUEST_DS_LIMIT,
588	VMX_VMCS32_GUEST_FS_LIMIT,
589	VMX_VMCS32_GUEST_GS_LIMIT,
590	VMX_VMCS32_GUEST_LDTR_LIMIT,
591	VMX_VMCS32_GUEST_TR_LIMIT,
592	VMX_VMCS32_GUEST_GDTR_LIMIT,
593	VMX_VMCS32_GUEST_IDTR_LIMIT,
594	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
595	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
596	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
597	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
598	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
599	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
600	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
602	VMX_VMCS32_GUEST_INT_STATE,
603	VMX_VMCS32_GUEST_ACTIVITY_STATE,
604	VMX_VMCS32_GUEST_SMBASE,
605	VMX_VMCS32_GUEST_SYSENTER_CS,
606	VMX_VMCS32_PREEMPT_TIMER_VALUE,
607
608	/* 32-bit host-state fields. */
609	VMX_VMCS32_HOST_SYSENTER_CS,
610
611	/* Natural-width control fields. */
612	VMX_VMCS_CTRL_CR0_MASK,
613	VMX_VMCS_CTRL_CR4_MASK,
614	VMX_VMCS_CTRL_CR0_READ_SHADOW,
615	VMX_VMCS_CTRL_CR4_READ_SHADOW,
616	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
617	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
618	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
619	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
620
621	/* Natural-width read-only data fields. */
622	VMX_VMCS_RO_EXIT_QUALIFICATION,
623	VMX_VMCS_RO_IO_RCX,
624	VMX_VMCS_RO_IO_RSI,
625	VMX_VMCS_RO_IO_RDI,
626	VMX_VMCS_RO_IO_RIP,
627	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
628
629	/* Natural-width guest-state field */
630	VMX_VMCS_GUEST_CR0,
631	VMX_VMCS_GUEST_CR3,
632	VMX_VMCS_GUEST_CR4,
633	VMX_VMCS_GUEST_ES_BASE,
634	VMX_VMCS_GUEST_CS_BASE,
635	VMX_VMCS_GUEST_SS_BASE,
636	VMX_VMCS_GUEST_DS_BASE,
637	VMX_VMCS_GUEST_FS_BASE,
638	VMX_VMCS_GUEST_GS_BASE,
639	VMX_VMCS_GUEST_LDTR_BASE,
640	VMX_VMCS_GUEST_TR_BASE,
641	VMX_VMCS_GUEST_GDTR_BASE,
642	VMX_VMCS_GUEST_IDTR_BASE,
643	VMX_VMCS_GUEST_DR7,
644	VMX_VMCS_GUEST_RSP,
645	VMX_VMCS_GUEST_RIP,
646	VMX_VMCS_GUEST_RFLAGS,
647	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
648	VMX_VMCS_GUEST_SYSENTER_ESP,
649	VMX_VMCS_GUEST_SYSENTER_EIP,
650
651	/* Natural-width host-state fields */
652	VMX_VMCS_HOST_CR0,
653	VMX_VMCS_HOST_CR3,
654	VMX_VMCS_HOST_CR4,
655	VMX_VMCS_HOST_FS_BASE,
656	VMX_VMCS_HOST_GS_BASE,
657	VMX_VMCS_HOST_TR_BASE,
658	VMX_VMCS_HOST_GDTR_BASE,
659	VMX_VMCS_HOST_IDTR_BASE,
660	VMX_VMCS_HOST_SYSENTER_ESP,
661	VMX_VMCS_HOST_SYSENTER_EIP,
662	VMX_VMCS_HOST_RSP,
663	VMX_VMCS_HOST_RIP
664	};
665	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
666
667	static const uint32_t g_aVmcsSegBase[] =
668	{
669	VMX_VMCS_GUEST_ES_BASE,
670	VMX_VMCS_GUEST_CS_BASE,
671	VMX_VMCS_GUEST_SS_BASE,
672	VMX_VMCS_GUEST_DS_BASE,
673	VMX_VMCS_GUEST_FS_BASE,
674	VMX_VMCS_GUEST_GS_BASE
675	};
676	static const uint32_t g_aVmcsSegSel[] =
677	{
678	VMX_VMCS16_GUEST_ES_SEL,
679	VMX_VMCS16_GUEST_CS_SEL,
680	VMX_VMCS16_GUEST_SS_SEL,
681	VMX_VMCS16_GUEST_DS_SEL,
682	VMX_VMCS16_GUEST_FS_SEL,
683	VMX_VMCS16_GUEST_GS_SEL
684	};
685	static const uint32_t g_aVmcsSegLimit[] =
686	{
687	VMX_VMCS32_GUEST_ES_LIMIT,
688	VMX_VMCS32_GUEST_CS_LIMIT,
689	VMX_VMCS32_GUEST_SS_LIMIT,
690	VMX_VMCS32_GUEST_DS_LIMIT,
691	VMX_VMCS32_GUEST_FS_LIMIT,
692	VMX_VMCS32_GUEST_GS_LIMIT
693	};
694	static const uint32_t g_aVmcsSegAttr[] =
695	{
696	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
697	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
698	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
699	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
700	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
701	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
702	};
703	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
704	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
705	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
706	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
707
708	#ifdef HMVMX_USE_FUNCTION_TABLE
709	/**
710	* VMX_EXIT dispatch table.
711	*/
712	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
713	{
714	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
715	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
716	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
717	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
718	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
719	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
720	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
721	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
722	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
723	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
724	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
725	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
726	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
727	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
728	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
729	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
730	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
731	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
732	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
733	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
734	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
735	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
736	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
737	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
738	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
739	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
740	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
741	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
742	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
743	#else
744	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
745	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
746	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
747	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
748	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
749	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
750	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
751	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
752	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
753	#endif
754	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
755	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
756	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
757	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
758	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
759	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
760	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
761	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
762	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
763	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
764	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
765	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
766	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
767	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
768	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
769	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
770	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
771	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
772	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
773	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
774	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
775	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
776	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
777	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
778	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
779	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
780	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
781	#else
782	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
783	#endif
784	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
785	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
786	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
787	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
788	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
789	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
790	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
791	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
792	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
793	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
794	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
795	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
796	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
797	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
798	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
799	};
800	#endif /* HMVMX_USE_FUNCTION_TABLE */
801
802	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
803	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
804	{
805	/* 0 */ "(Not Used)",
806	/* 1 */ "VMCALL executed in VMX root operation.",
807	/* 2 */ "VMCLEAR with invalid physical address.",
808	/* 3 */ "VMCLEAR with VMXON pointer.",
809	/* 4 */ "VMLAUNCH with non-clear VMCS.",
810	/* 5 */ "VMRESUME with non-launched VMCS.",
811	/* 6 */ "VMRESUME after VMXOFF",
812	/* 7 */ "VM-entry with invalid control fields.",
813	/* 8 */ "VM-entry with invalid host state fields.",
814	/* 9 */ "VMPTRLD with invalid physical address.",
815	/* 10 */ "VMPTRLD with VMXON pointer.",
816	/* 11 */ "VMPTRLD with incorrect revision identifier.",
817	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
818	/* 13 */ "VMWRITE to read-only VMCS component.",
819	/* 14 */ "(Not Used)",
820	/* 15 */ "VMXON executed in VMX root operation.",
821	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
822	/* 17 */ "VM-entry with non-launched executing VMCS.",
823	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
824	/* 19 */ "VMCALL with non-clear VMCS.",
825	/* 20 */ "VMCALL with invalid VM-exit control fields.",
826	/* 21 */ "(Not Used)",
827	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
828	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
829	/* 24 */ "VMCALL with invalid SMM-monitor features.",
830	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
831	/* 26 */ "VM-entry with events blocked by MOV SS.",
832	/* 27 */ "(Not Used)",
833	/* 28 */ "Invalid operand to INVEPT/INVVPID."
834	};
835	#endif /* VBOX_STRICT && LOG_ENABLED */
836
837
838	/**
839	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
840	*
841	* Any bit set in this mask is owned by the host/hypervisor and would cause a
842	* VM-exit when modified by the guest.
843	*
844	* @returns The static CR0 guest/host mask.
845	* @param pVCpu The cross context virtual CPU structure.
846	*/
847	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
848	{
849	/*
850	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
851	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
852	*/
853	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
854	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
855	* and @bugref{6944}. */
856	PVM pVM = pVCpu->CTX_SUFF(pVM);
857	return ( X86_CR0_PE
858	\| X86_CR0_NE
859	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
860	\| X86_CR0_PG
861	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
862	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
863	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
864	}
865
866
867	/**
868	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
869	*
870	* Any bit set in this mask is owned by the host/hypervisor and would cause a
871	* VM-exit when modified by the guest.
872	*
873	* @returns The static CR4 guest/host mask.
874	* @param pVCpu The cross context virtual CPU structure.
875	*/
876	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
877	{
878	/*
879	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
880	* these bits are reserved on hardware that does not support them. Since the
881	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
882	* these bits and handle it depending on whether we expose them to the guest.
883	*/
884	PVM pVM = pVCpu->CTX_SUFF(pVM);
885	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
886	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
887	return ( X86_CR4_VMXE
888	\| X86_CR4_VME
889	\| X86_CR4_PAE
890	\| X86_CR4_PGE
891	\| X86_CR4_PSE
892	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
893	\| (fPcid ? X86_CR4_PCIDE : 0));
894	}
895
896
897	/**
898	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
899	* area.
900	*
901	* @returns @c true if it's different, @c false otherwise.
902	* @param pVmcsInfo The VMCS info. object.
903	*/
904	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
905	{
906	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
907	&& pVmcsInfo->pvGuestMsrStore);
908	}
909
910
911	/**
912	* Sets the given Processor-based VM-execution controls.
913	*
914	* @param pVmxTransient The VMX-transient structure.
915	* @param uProcCtls The Processor-based VM-execution controls to set.
916	*/
917	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
918	{
919	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
920	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
921	{
922	pVmcsInfo->u32ProcCtls \|= uProcCtls;
923	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
924	AssertRC(rc);
925	}
926	}
927
928
929	/**
930	* Removes the given Processor-based VM-execution controls.
931	*
932	* @param pVCpu The cross context virtual CPU structure.
933	* @param pVmxTransient The VMX-transient structure.
934	* @param uProcCtls The Processor-based VM-execution controls to remove.
935	*
936	* @remarks When executing a nested-guest, this will not remove any of the specified
937	* controls if the guest hypervisor has set any one of them.
938	*/
939	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
940	{
941	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
942	if (pVmcsInfo->u32ProcCtls & uProcCtls)
943	{
944	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
945	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
946	? true
947	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
948	#else
949	NOREF(pVCpu);
950	bool const fRemoveCtls = true;
951	#endif
952	if (fRemoveCtls)
953	{
954	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
955	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
956	AssertRC(rc);
957	}
958	}
959	}
960
961
962	/**
963	* Sets the TSC offset for the current VMCS.
964	*
965	* @param uTscOffset The TSC offset to set.
966	* @param pVmcsInfo The VMCS info. object.
967	*/
968	static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
969	{
970	if (pVmcsInfo->u64TscOffset != uTscOffset)
971	{
972	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
973	AssertRC(rc);
974	pVmcsInfo->u64TscOffset = uTscOffset;
975	}
976	}
977
978
979	/**
980	* Adds one or more exceptions to the exception bitmap and commits it to the current
981	* VMCS.
982	*
983	* @returns VBox status code.
984	* @param pVmxTransient The VMX-transient structure.
985	* @param uXcptMask The exception(s) to add.
986	*/
987	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
988	{
989	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
990	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
991	if ((uXcptBitmap & uXcptMask) != uXcptMask)
992	{
993	uXcptBitmap \|= uXcptMask;
994	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
995	AssertRCReturn(rc, rc);
996	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
997	}
998	return VINF_SUCCESS;
999	}
1000
1001
1002	/**
1003	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1004	*
1005	* @returns VBox status code.
1006	* @param pVmxTransient The VMX-transient structure.
1007	* @param uXcpt The exception to add.
1008	*/
1009	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1010	{
1011	Assert(uXcpt <= X86_XCPT_LAST);
1012	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1013	}
1014
1015
1016	/**
1017	* Remove one or more exceptions from the exception bitmap and commits it to the
1018	* current VMCS.
1019	*
1020	* This takes care of not removing the exception intercept if a nested-guest
1021	* requires the exception to be intercepted.
1022	*
1023	* @returns VBox status code.
1024	* @param pVCpu The cross context virtual CPU structure.
1025	* @param pVmxTransient The VMX-transient structure.
1026	* @param uXcptMask The exception(s) to remove.
1027	*/
1028	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1029	{
1030	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1031	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1032	if (u32XcptBitmap & uXcptMask)
1033	{
1034	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1035	if (!pVmxTransient->fIsNestedGuest)
1036	{ /* likely */ }
1037	else
1038	{
1039	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1040	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1041	}
1042	#endif
1043	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1044	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1045	\| RT_BIT(X86_XCPT_DE)
1046	\| RT_BIT(X86_XCPT_NM)
1047	\| RT_BIT(X86_XCPT_TS)
1048	\| RT_BIT(X86_XCPT_UD)
1049	\| RT_BIT(X86_XCPT_NP)
1050	\| RT_BIT(X86_XCPT_SS)
1051	\| RT_BIT(X86_XCPT_GP)
1052	\| RT_BIT(X86_XCPT_PF)
1053	\| RT_BIT(X86_XCPT_MF));
1054	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1055	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1056	#endif
1057	if (uXcptMask)
1058	{
1059	/* Validate we are not removing any essential exception intercepts. */
1060	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1061	NOREF(pVCpu);
1062	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1063	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1064
1065	/* Remove it from the exception bitmap. */
1066	u32XcptBitmap &= ~uXcptMask;
1067
1068	/* Commit and update the cache if necessary. */
1069	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1070	{
1071	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1072	AssertRCReturn(rc, rc);
1073	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1074	}
1075	}
1076	}
1077	return VINF_SUCCESS;
1078	}
1079
1080
1081	/**
1082	* Remove an exceptions from the exception bitmap and commits it to the current
1083	* VMCS.
1084	*
1085	* @returns VBox status code.
1086	* @param pVCpu The cross context virtual CPU structure.
1087	* @param pVmxTransient The VMX-transient structure.
1088	* @param uXcpt The exception to remove.
1089	*/
1090	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1091	{
1092	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1093	}
1094
1095
1096	/**
1097	* Loads the VMCS specified by the VMCS info. object.
1098	*
1099	* @returns VBox status code.
1100	* @param pVmcsInfo The VMCS info. object.
1101	*
1102	* @remarks Can be called with interrupts disabled.
1103	*/
1104	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1105	{
1106	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1107	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1108
1109	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1110	if (RT_SUCCESS(rc))
1111	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1112	return rc;
1113	}
1114
1115
1116	/**
1117	* Clears the VMCS specified by the VMCS info. object.
1118	*
1119	* @returns VBox status code.
1120	* @param pVmcsInfo The VMCS info. object.
1121	*
1122	* @remarks Can be called with interrupts disabled.
1123	*/
1124	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1125	{
1126	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1127	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1128
1129	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1130	if (RT_SUCCESS(rc))
1131	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1132	return rc;
1133	}
1134
1135
1136	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1137	/**
1138	* Loads the shadow VMCS specified by the VMCS info. object.
1139	*
1140	* @returns VBox status code.
1141	* @param pVmcsInfo The VMCS info. object.
1142	*
1143	* @remarks Can be called with interrupts disabled.
1144	*/
1145	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1146	{
1147	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1148	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1149
1150	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1151	if (RT_SUCCESS(rc))
1152	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1153	return rc;
1154	}
1155
1156
1157	/**
1158	* Clears the shadow VMCS specified by the VMCS info. object.
1159	*
1160	* @returns VBox status code.
1161	* @param pVmcsInfo The VMCS info. object.
1162	*
1163	* @remarks Can be called with interrupts disabled.
1164	*/
1165	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1166	{
1167	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1168	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1169
1170	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1171	if (RT_SUCCESS(rc))
1172	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1173	return rc;
1174	}
1175
1176
1177	/**
1178	* Switches from and to the specified VMCSes.
1179	*
1180	* @returns VBox status code.
1181	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1182	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1183	*
1184	* @remarks Called with interrupts disabled.
1185	*/
1186	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1187	{
1188	/*
1189	* Clear the VMCS we are switching out if it has not already been cleared.
1190	* This will sync any CPU internal data back to the VMCS.
1191	*/
1192	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1193	{
1194	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1195	if (RT_SUCCESS(rc))
1196	{
1197	/*
1198	* The shadow VMCS, if any, would not be active at this point since we
1199	* would have cleared it while importing the virtual hardware-virtualization
1200	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1201	* clear the shadow VMCS here, just assert for safety.
1202	*/
1203	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1204	}
1205	else
1206	return rc;
1207	}
1208
1209	/*
1210	* Clear the VMCS we are switching to if it has not already been cleared.
1211	* This will initialize the VMCS launch state to "clear" required for loading it.
1212	*
1213	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1214	*/
1215	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1216	{
1217	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1218	if (RT_SUCCESS(rc))
1219	{ /* likely */ }
1220	else
1221	return rc;
1222	}
1223
1224	/*
1225	* Finally, load the VMCS we are switching to.
1226	*/
1227	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1228	}
1229
1230
1231	/**
1232	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1233	* caller.
1234	*
1235	* @returns VBox status code.
1236	* @param pVCpu The cross context virtual CPU structure.
1237	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1238	* true) or guest VMCS (pass false).
1239	*/
1240	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPU pVCpu, bool fSwitchToNstGstVmcs)
1241	{
1242	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1243	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1244
1245	PVMXVMCSINFO pVmcsInfoFrom;
1246	PVMXVMCSINFO pVmcsInfoTo;
1247	if (fSwitchToNstGstVmcs)
1248	{
1249	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1250	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1251	}
1252	else
1253	{
1254	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1255	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1256	}
1257
1258	/*
1259	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1260	* preemption hook code path acquires the current VMCS.
1261	*/
1262	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1263
1264	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1265	if (RT_SUCCESS(rc))
1266	{
1267	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1268
1269	/*
1270	* If we are switching to a VMCS that was executed on a different host CPU or was
1271	* never executed before, flag that we need to export the host state before executing
1272	* guest/nested-guest code using hardware-assisted VMX.
1273	*
1274	* This could probably be done in a preemptible context since the preemption hook
1275	* will flag the necessary change in host context. However, since preemption is
1276	* already disabled and to avoid making assumptions about host specific code in
1277	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1278	* disabled.
1279	*/
1280	if (pVmcsInfoTo->idHostCpu == RTMpCpuId())
1281	{ /* likely */ }
1282	else
1283	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1284
1285	ASMSetFlags(fEFlags);
1286
1287	/*
1288	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1289	* flag that we need to update the host MSR values there. Even if we decide in the
1290	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1291	* if its content differs, we would have to update the host MSRs anyway.
1292	*/
1293	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1294	}
1295	else
1296	ASMSetFlags(fEFlags);
1297	return rc;
1298	}
1299	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1300
1301
1302	/**
1303	* Updates the VM's last error record.
1304	*
1305	* If there was a VMX instruction error, reads the error data from the VMCS and
1306	* updates VCPU's last error record as well.
1307	*
1308	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1309	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1310	* VERR_VMX_INVALID_VMCS_FIELD.
1311	* @param rc The error code.
1312	*/
1313	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1314	{
1315	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1316	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1317	{
1318	AssertPtrReturnVoid(pVCpu);
1319	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1320	}
1321	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1322	}
1323
1324
1325	#ifdef VBOX_STRICT
1326	/**
1327	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1328	* transient structure.
1329	*
1330	* @returns VBox status code.
1331	* @param pVmxTransient The VMX-transient structure.
1332	*
1333	* @remarks No-long-jump zone!!!
1334	*/
1335	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1336	{
1337	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1338	AssertRCReturn(rc, rc);
1339	return VINF_SUCCESS;
1340	}
1341
1342
1343	/**
1344	* Reads the VM-entry exception error code field from the VMCS into
1345	* the VMX transient structure.
1346	*
1347	* @returns VBox status code.
1348	* @param pVmxTransient The VMX-transient structure.
1349	*
1350	* @remarks No-long-jump zone!!!
1351	*/
1352	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1353	{
1354	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1355	AssertRCReturn(rc, rc);
1356	return VINF_SUCCESS;
1357	}
1358
1359
1360	/**
1361	* Reads the VM-entry exception error code field from the VMCS into
1362	* the VMX transient structure.
1363	*
1364	* @returns VBox status code.
1365	* @param pVmxTransient The VMX-transient structure.
1366	*
1367	* @remarks No-long-jump zone!!!
1368	*/
1369	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1370	{
1371	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1372	AssertRCReturn(rc, rc);
1373	return VINF_SUCCESS;
1374	}
1375	#endif /* VBOX_STRICT */
1376
1377
1378	/**
1379	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1380	* transient structure.
1381	*
1382	* @returns VBox status code.
1383	* @param pVmxTransient The VMX-transient structure.
1384	*/
1385	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1386	{
1387	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1388	{
1389	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1390	AssertRCReturn(rc,rc);
1391	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1392	}
1393	return VINF_SUCCESS;
1394	}
1395
1396
1397	/**
1398	* Reads the VM-exit interruption error code from the VMCS into the VMX
1399	* transient structure.
1400	*
1401	* @returns VBox status code.
1402	* @param pVmxTransient The VMX-transient structure.
1403	*/
1404	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1405	{
1406	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1407	{
1408	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1409	AssertRCReturn(rc, rc);
1410	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1411	}
1412	return VINF_SUCCESS;
1413	}
1414
1415
1416	/**
1417	* Reads the VM-exit instruction length field from the VMCS into the VMX
1418	* transient structure.
1419	*
1420	* @returns VBox status code.
1421	* @param pVmxTransient The VMX-transient structure.
1422	*/
1423	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1424	{
1425	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1426	{
1427	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1428	AssertRCReturn(rc, rc);
1429	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1430	}
1431	return VINF_SUCCESS;
1432	}
1433
1434
1435	/**
1436	* Reads the VM-exit instruction-information field from the VMCS into
1437	* the VMX transient structure.
1438	*
1439	* @returns VBox status code.
1440	* @param pVmxTransient The VMX-transient structure.
1441	*/
1442	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1443	{
1444	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1445	{
1446	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1447	AssertRCReturn(rc, rc);
1448	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1449	}
1450	return VINF_SUCCESS;
1451	}
1452
1453
1454	/**
1455	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1456	*
1457	* @returns VBox status code.
1458	* @param pVmxTransient The VMX-transient structure.
1459	*/
1460	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMXTRANSIENT pVmxTransient)
1461	{
1462	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1463	{
1464	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1465	AssertRCReturn(rc, rc);
1466	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1467	}
1468	return VINF_SUCCESS;
1469	}
1470
1471
1472	/**
1473	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1474	*
1475	* @returns VBox status code.
1476	* @param pVmxTransient The VMX-transient structure.
1477	*/
1478	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMXTRANSIENT pVmxTransient)
1479	{
1480	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1481	{
1482	int rc = VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1483	AssertRCReturn(rc, rc);
1484	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1485	}
1486	return VINF_SUCCESS;
1487	}
1488
1489
1490	/**
1491	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1492	*
1493	* @returns VBox status code.
1494	* @param pVmxTransient The VMX-transient structure.
1495	*/
1496	DECLINLINE(int) hmR0VmxReadGuestPhysicalAddrVmcs(PVMXTRANSIENT pVmxTransient)
1497	{
1498	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1499	{
1500	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1501	AssertRCReturn(rc, rc);
1502	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1503	}
1504	return VINF_SUCCESS;
1505	}
1506
1507
1508	/**
1509	* Reads the IDT-vectoring information field from the VMCS into the VMX
1510	* transient structure.
1511	*
1512	* @returns VBox status code.
1513	* @param pVmxTransient The VMX-transient structure.
1514	*
1515	* @remarks No-long-jump zone!!!
1516	*/
1517	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1518	{
1519	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1520	{
1521	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1522	AssertRCReturn(rc, rc);
1523	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1524	}
1525	return VINF_SUCCESS;
1526	}
1527
1528
1529	/**
1530	* Reads the IDT-vectoring error code from the VMCS into the VMX
1531	* transient structure.
1532	*
1533	* @returns VBox status code.
1534	* @param pVmxTransient The VMX-transient structure.
1535	*/
1536	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1537	{
1538	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1539	{
1540	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1541	AssertRCReturn(rc, rc);
1542	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1543	}
1544	return VINF_SUCCESS;
1545	}
1546
1547	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1548	/**
1549	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1550	*
1551	* @returns VBox status code.
1552	* @param pVmxTransient The VMX-transient structure.
1553	*/
1554	static int hmR0VmxReadAllRoFieldsVmcs(PVMXTRANSIENT pVmxTransient)
1555	{
1556	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1557	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1558	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1559	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1560	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1561	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1562	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1563	rc \|= VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1564	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1565	AssertRCReturn(rc, rc);
1566	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1567	\| HMVMX_READ_EXIT_INSTR_LEN
1568	\| HMVMX_READ_EXIT_INSTR_INFO
1569	\| HMVMX_READ_IDT_VECTORING_INFO
1570	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1571	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1572	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1573	\| HMVMX_READ_GUEST_LINEAR_ADDR
1574	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1575	return VINF_SUCCESS;
1576	}
1577	#endif
1578
1579	/**
1580	* Enters VMX root mode operation on the current CPU.
1581	*
1582	* @returns VBox status code.
1583	* @param pVM The cross context VM structure. Can be
1584	* NULL, after a resume.
1585	* @param HCPhysCpuPage Physical address of the VMXON region.
1586	* @param pvCpuPage Pointer to the VMXON region.
1587	*/
1588	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1589	{
1590	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1591	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1592	Assert(pvCpuPage);
1593	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1594
1595	if (pVM)
1596	{
1597	/* Write the VMCS revision identifier to the VMXON region. */
1598	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1599	}
1600
1601	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1602	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1603
1604	/* Enable the VMX bit in CR4 if necessary. */
1605	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1606
1607	/* Enter VMX root mode. */
1608	int rc = VMXEnable(HCPhysCpuPage);
1609	if (RT_FAILURE(rc))
1610	{
1611	if (!(uOldCr4 & X86_CR4_VMXE))
1612	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1613
1614	if (pVM)
1615	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1616	}
1617
1618	/* Restore interrupts. */
1619	ASMSetFlags(fEFlags);
1620	return rc;
1621	}
1622
1623
1624	/**
1625	* Exits VMX root mode operation on the current CPU.
1626	*
1627	* @returns VBox status code.
1628	*/
1629	static int hmR0VmxLeaveRootMode(void)
1630	{
1631	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1632
1633	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1634	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1635
1636	/* If we're for some reason not in VMX root mode, then don't leave it. */
1637	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1638
1639	int rc;
1640	if (uHostCr4 & X86_CR4_VMXE)
1641	{
1642	/* Exit VMX root mode and clear the VMX bit in CR4. */
1643	VMXDisable();
1644	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1645	rc = VINF_SUCCESS;
1646	}
1647	else
1648	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1649
1650	/* Restore interrupts. */
1651	ASMSetFlags(fEFlags);
1652	return rc;
1653	}
1654
1655
1656	/**
1657	* Allocates and maps a physically contiguous page. The allocated page is
1658	* zero'd out (used by various VT-x structures).
1659	*
1660	* @returns IPRT status code.
1661	* @param pMemObj Pointer to the ring-0 memory object.
1662	* @param ppVirt Where to store the virtual address of the allocation.
1663	* @param pHCPhys Where to store the physical address of the allocation.
1664	*/
1665	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1666	{
1667	AssertPtr(pMemObj);
1668	AssertPtr(ppVirt);
1669	AssertPtr(pHCPhys);
1670	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1671	if (RT_FAILURE(rc))
1672	return rc;
1673	ppVirt = RTR0MemObjAddress(pMemObj);
1674	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1675	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1676	return VINF_SUCCESS;
1677	}
1678
1679
1680	/**
1681	* Frees and unmaps an allocated, physical page.
1682	*
1683	* @param pMemObj Pointer to the ring-0 memory object.
1684	* @param ppVirt Where to re-initialize the virtual address of allocation as
1685	* 0.
1686	* @param pHCPhys Where to re-initialize the physical address of the
1687	* allocation as 0.
1688	*/
1689	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1690	{
1691	AssertPtr(pMemObj);
1692	AssertPtr(ppVirt);
1693	AssertPtr(pHCPhys);
1694	/* NULL is valid, accepted and ignored by the free function below. */
1695	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1696	*pMemObj = NIL_RTR0MEMOBJ;
1697	*ppVirt = NULL;
1698	*pHCPhys = NIL_RTHCPHYS;
1699	}
1700
1701
1702	/**
1703	* Initializes a VMCS info. object.
1704	*
1705	* @param pVmcsInfo The VMCS info. object.
1706	*/
1707	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1708	{
1709	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1710
1711	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1712	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1713	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1714	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1715	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1716	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1717	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1718	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1719	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1720	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1721	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1722	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1723	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1724	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1725	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1726	pVmcsInfo->idHostCpu = NIL_RTCPUID;
1727	}
1728
1729
1730	/**
1731	* Frees the VT-x structures for a VMCS info. object.
1732	*
1733	* @param pVM The cross context VM structure.
1734	* @param pVmcsInfo The VMCS info. object.
1735	*/
1736	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1737	{
1738	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1739
1740	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1741	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1742	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1743	#endif
1744
1745	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1746	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1747
1748	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1749	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1750	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1751
1752	hmR0VmxInitVmcsInfo(pVmcsInfo);
1753	}
1754
1755
1756	/**
1757	* Allocates the VT-x structures for a VMCS info. object.
1758	*
1759	* @returns VBox status code.
1760	* @param pVCpu The cross context virtual CPU structure.
1761	* @param pVmcsInfo The VMCS info. object.
1762	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1763	*/
1764	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1765	{
1766	PVM pVM = pVCpu->CTX_SUFF(pVM);
1767
1768	/* Allocate the guest VM control structure (VMCS). */
1769	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1770	if (RT_SUCCESS(rc))
1771	{
1772	if (!fIsNstGstVmcs)
1773	{
1774	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1775	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1776	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1777	#endif
1778	if (RT_SUCCESS(rc))
1779	{
1780	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1781	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1782	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1783	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1784	}
1785	}
1786	else
1787	{
1788	/* We don't yet support exposing VMCS shadowing to the guest. */
1789	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1790	Assert(!pVmcsInfo->pvShadowVmcs);
1791
1792	/* Get the allocated virtual-APIC page from CPUM. */
1793	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1794	{
1795	/** @todo NSTVMX: Get rid of this. There is no need to allocate a separate HC
1796	* page for this. Use the one provided by the nested-guest directly. */
1797	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1798	&pVmcsInfo->HCPhysVirtApic);
1799	Assert(pVmcsInfo->pbVirtApic);
1800	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1801	}
1802	}
1803
1804	if (RT_SUCCESS(rc))
1805	{
1806	/*
1807	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1808	* transparent accesses of specific MSRs.
1809	*
1810	* If the condition for enabling MSR bitmaps changes here, don't forget to
1811	* update HMIsMsrBitmapActive().
1812	*
1813	* We don't share MSR bitmaps between the guest and nested-guest as we then
1814	* don't need to care about carefully restoring the guest MSR bitmap.
1815	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1816	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1817	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1818	* we do that later while merging VMCS.
1819	*/
1820	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1821	{
1822	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1823	if ( RT_SUCCESS(rc)
1824	&& !fIsNstGstVmcs)
1825	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1826	}
1827
1828	if (RT_SUCCESS(rc))
1829	{
1830	/*
1831	* Allocate the VM-entry MSR-load area for the guest MSRs.
1832	*
1833	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1834	* the guest and nested-guest.
1835	*/
1836	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1837	&pVmcsInfo->HCPhysGuestMsrLoad);
1838	if (RT_SUCCESS(rc))
1839	{
1840	/*
1841	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1842	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1843	*/
1844	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1845	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1846	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1847
1848	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1849	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1850	&pVmcsInfo->HCPhysHostMsrLoad);
1851	}
1852	}
1853	}
1854	}
1855
1856	return rc;
1857	}
1858
1859
1860	/**
1861	* Free all VT-x structures for the VM.
1862	*
1863	* @returns IPRT status code.
1864	* @param pVM The cross context VM structure.
1865	*/
1866	static void hmR0VmxStructsFree(PVM pVM)
1867	{
1868	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1869	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1870	#endif
1871	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1872
1873	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1874	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1875	{
1876	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1877	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1878	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1879	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1880	}
1881	#endif
1882
1883	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1884	{
1885	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1886	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1887	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1888	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1889	if (pVM->cpum.ro.GuestFeatures.fVmx)
1890	{
1891	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1892	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1893	}
1894	#endif
1895	}
1896	}
1897
1898
1899	/**
1900	* Allocate all VT-x structures for the VM.
1901	*
1902	* @returns IPRT status code.
1903	* @param pVM The cross context VM structure.
1904	*/
1905	static int hmR0VmxStructsAlloc(PVM pVM)
1906	{
1907	/*
1908	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1909	* The VMCS size cannot be more than 4096 bytes.
1910	*
1911	* See Intel spec. Appendix A.1 "Basic VMX Information".
1912	*/
1913	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1914	if (cbVmcs <= X86_PAGE_4K_SIZE)
1915	{ /* likely */ }
1916	else
1917	{
1918	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1919	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1920	}
1921
1922	/*
1923	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1924	*/
1925	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1926	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1927	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1928	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1929	#endif
1930
1931	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1932	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1933	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1934
1935	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1936	{
1937	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1938	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1939	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1940	}
1941
1942	/*
1943	* Allocate per-VM VT-x structures.
1944	*/
1945	int rc = VINF_SUCCESS;
1946	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1947	/* Allocate crash-dump magic scratch page. */
1948	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1949	if (RT_FAILURE(rc))
1950	{
1951	hmR0VmxStructsFree(pVM);
1952	return rc;
1953	}
1954	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1955	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1956	#endif
1957
1958	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1959	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1960	{
1961	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1962	&pVM->hm.s.vmx.HCPhysApicAccess);
1963	if (RT_FAILURE(rc))
1964	{
1965	hmR0VmxStructsFree(pVM);
1966	return rc;
1967	}
1968	}
1969
1970	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1971	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1972	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1973	{
1974	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1975	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1976	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1977	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1978	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1979	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1980	{
1981	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1982	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1983	if (RT_SUCCESS(rc))
1984	{
1985	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1986	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1987	}
1988	}
1989	else
1990	rc = VERR_NO_MEMORY;
1991
1992	if (RT_FAILURE(rc))
1993	{
1994	hmR0VmxStructsFree(pVM);
1995	return rc;
1996	}
1997	}
1998	#endif
1999
2000	/*
2001	* Initialize per-VCPU VT-x structures.
2002	*/
2003	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2004	{
2005	/* Allocate the guest VMCS structures. */
2006	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2007	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2008	if (RT_SUCCESS(rc))
2009	{
2010	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2011	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2012	if (pVM->cpum.ro.GuestFeatures.fVmx)
2013	{
2014	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2015	if (RT_SUCCESS(rc))
2016	{ /* likely */ }
2017	else
2018	break;
2019	}
2020	#endif
2021	}
2022	else
2023	break;
2024	}
2025
2026	if (RT_FAILURE(rc))
2027	{
2028	hmR0VmxStructsFree(pVM);
2029	return rc;
2030	}
2031
2032	return VINF_SUCCESS;
2033	}
2034
2035	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2036	/**
2037	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2038	*
2039	* @returns @c true if the MSR is intercepted, @c false otherwise.
2040	* @param pvMsrBitmap The MSR bitmap.
2041	* @param offMsr The MSR byte offset.
2042	* @param iBit The bit offset from the byte offset.
2043	*/
2044	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2045	{
2046	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2047	Assert(pbMsrBitmap);
2048	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2049	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2050	}
2051	#endif
2052
2053	/**
2054	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2055	*
2056	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2057	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2058	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2059	* the read/write access of this MSR.
2060	*
2061	* @param pVCpu The cross context virtual CPU structure.
2062	* @param pVmcsInfo The VMCS info. object.
2063	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2064	* @param idMsr The MSR value.
2065	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2066	* include both a read -and- a write permission!
2067	*
2068	* @sa CPUMGetVmxMsrPermission.
2069	* @remarks Can be called with interrupts disabled.
2070	*/
2071	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2072	{
2073	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2074	Assert(pbMsrBitmap);
2075	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2076
2077	/*
2078	* MSR-bitmap Layout:
2079	* Byte index MSR range Interpreted as
2080	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2081	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2082	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2083	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2084	*
2085	* A bit corresponding to an MSR within the above range causes a VM-exit
2086	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2087	* the MSR range, it always cause a VM-exit.
2088	*
2089	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2090	*/
2091	uint16_t const offBitmapRead = 0;
2092	uint16_t const offBitmapWrite = 0x800;
2093	uint16_t offMsr;
2094	int32_t iBit;
2095	if (idMsr <= UINT32_C(0x00001fff))
2096	{
2097	offMsr = 0;
2098	iBit = idMsr;
2099	}
2100	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2101	{
2102	offMsr = 0x400;
2103	iBit = idMsr - UINT32_C(0xc0000000);
2104	}
2105	else
2106	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2107
2108	/*
2109	* Set the MSR read permission.
2110	*/
2111	uint16_t const offMsrRead = offBitmapRead + offMsr;
2112	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2113	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2114	{
2115	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2116	bool const fClear = !fIsNstGstVmcs ? true
2117	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2118	#else
2119	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2120	bool const fClear = true;
2121	#endif
2122	if (fClear)
2123	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2124	}
2125	else
2126	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2127
2128	/*
2129	* Set the MSR write permission.
2130	*/
2131	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2132	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2133	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2134	{
2135	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2136	bool const fClear = !fIsNstGstVmcs ? true
2137	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2138	#else
2139	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2140	bool const fClear = true;
2141	#endif
2142	if (fClear)
2143	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2144	}
2145	else
2146	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2147	}
2148
2149
2150	/**
2151	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2152	* area.
2153	*
2154	* @returns VBox status code.
2155	* @param pVCpu The cross context virtual CPU structure.
2156	* @param pVmcsInfo The VMCS info. object.
2157	* @param cMsrs The number of MSRs.
2158	*/
2159	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2160	{
2161	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2162	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2163	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2164	{
2165	/* Commit the MSR counts to the VMCS and update the cache. */
2166	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2167	{
2168	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2169	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2170	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2171	AssertRCReturn(rc, rc);
2172
2173	pVmcsInfo->cEntryMsrLoad = cMsrs;
2174	pVmcsInfo->cExitMsrStore = cMsrs;
2175	pVmcsInfo->cExitMsrLoad = cMsrs;
2176	}
2177	return VINF_SUCCESS;
2178	}
2179
2180	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2181	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2182	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2183	}
2184
2185
2186	/**
2187	* Adds a new (or updates the value of an existing) guest/host MSR
2188	* pair to be swapped during the world-switch as part of the
2189	* auto-load/store MSR area in the VMCS.
2190	*
2191	* @returns VBox status code.
2192	* @param pVCpu The cross context virtual CPU structure.
2193	* @param pVmxTransient The VMX-transient structure.
2194	* @param idMsr The MSR.
2195	* @param uGuestMsrValue Value of the guest MSR.
2196	* @param fSetReadWrite Whether to set the guest read/write access of this
2197	* MSR (thus not causing a VM-exit).
2198	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2199	* necessary.
2200	*/
2201	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2202	bool fSetReadWrite, bool fUpdateHostMsr)
2203	{
2204	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2205	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2206	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2207	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2208	uint32_t i;
2209
2210	/* Paranoia. */
2211	Assert(pGuestMsrLoad);
2212
2213	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2214
2215	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2216	for (i = 0; i < cMsrs; i++)
2217	{
2218	if (pGuestMsrLoad[i].u32Msr == idMsr)
2219	break;
2220	}
2221
2222	bool fAdded = false;
2223	if (i == cMsrs)
2224	{
2225	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2226	++cMsrs;
2227	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2228	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2229
2230	/* Set the guest to read/write this MSR without causing VM-exits. */
2231	if ( fSetReadWrite
2232	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2233	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2234
2235	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2236	fAdded = true;
2237	}
2238
2239	/* Update the MSR value for the newly added or already existing MSR. */
2240	pGuestMsrLoad[i].u32Msr = idMsr;
2241	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2242
2243	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2244	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2245	{
2246	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2247	pGuestMsrStore[i].u32Msr = idMsr;
2248	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2249	}
2250
2251	/* Update the corresponding slot in the host MSR area. */
2252	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2253	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2254	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2255	pHostMsr[i].u32Msr = idMsr;
2256
2257	/*
2258	* Only if the caller requests to update the host MSR value AND we've newly added the
2259	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2260	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2261	*
2262	* We do this for performance reasons since reading MSRs may be quite expensive.
2263	*/
2264	if (fAdded)
2265	{
2266	if (fUpdateHostMsr)
2267	{
2268	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2269	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2270	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2271	}
2272	else
2273	{
2274	/* Someone else can do the work. */
2275	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2276	}
2277	}
2278	return VINF_SUCCESS;
2279	}
2280
2281
2282	/**
2283	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2284	* auto-load/store MSR area in the VMCS.
2285	*
2286	* @returns VBox status code.
2287	* @param pVCpu The cross context virtual CPU structure.
2288	* @param pVmxTransient The VMX-transient structure.
2289	* @param idMsr The MSR.
2290	*/
2291	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2292	{
2293	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2294	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2295	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2296	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2297
2298	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2299
2300	for (uint32_t i = 0; i < cMsrs; i++)
2301	{
2302	/* Find the MSR. */
2303	if (pGuestMsrLoad[i].u32Msr == idMsr)
2304	{
2305	/*
2306	* If it's the last MSR, we only need to reduce the MSR count.
2307	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2308	*/
2309	if (i < cMsrs - 1)
2310	{
2311	/* Remove it from the VM-entry MSR-load area. */
2312	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2313	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2314
2315	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2316	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2317	{
2318	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2319	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2320	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2321	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2322	}
2323
2324	/* Remove it from the VM-exit MSR-load area. */
2325	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2326	Assert(pHostMsr[i].u32Msr == idMsr);
2327	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2328	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2329	}
2330
2331	/* Reduce the count to reflect the removed MSR and bail. */
2332	--cMsrs;
2333	break;
2334	}
2335	}
2336
2337	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2338	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2339	{
2340	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2341	AssertRCReturn(rc, rc);
2342
2343	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2344	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2345	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2346
2347	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2348	return VINF_SUCCESS;
2349	}
2350
2351	return VERR_NOT_FOUND;
2352	}
2353
2354
2355	/**
2356	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2357	*
2358	* @returns @c true if found, @c false otherwise.
2359	* @param pVmcsInfo The VMCS info. object.
2360	* @param idMsr The MSR to find.
2361	*/
2362	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2363	{
2364	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2365	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2366	Assert(pMsrs);
2367	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2368	for (uint32_t i = 0; i < cMsrs; i++)
2369	{
2370	if (pMsrs[i].u32Msr == idMsr)
2371	return true;
2372	}
2373	return false;
2374	}
2375
2376
2377	/**
2378	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2379	*
2380	* @param pVCpu The cross context virtual CPU structure.
2381	* @param pVmcsInfo The VMCS info. object.
2382	*
2383	* @remarks No-long-jump zone!!!
2384	*/
2385	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2386	{
2387	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2388
2389	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2390	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2391	Assert(pHostMsrLoad);
2392	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2393	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2394	for (uint32_t i = 0; i < cMsrs; i++)
2395	{
2396	/*
2397	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2398	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2399	*/
2400	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2401	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2402	else
2403	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2404	}
2405	}
2406
2407
2408	/**
2409	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2410	* perform lazy restoration of the host MSRs while leaving VT-x.
2411	*
2412	* @param pVCpu The cross context virtual CPU structure.
2413	*
2414	* @remarks No-long-jump zone!!!
2415	*/
2416	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2417	{
2418	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2419
2420	/*
2421	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2422	*/
2423	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2424	{
2425	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2426	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2427	{
2428	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2429	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2430	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2431	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2432	}
2433	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2434	}
2435	}
2436
2437
2438	/**
2439	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2440	* lazily while leaving VT-x.
2441	*
2442	* @returns true if it does, false otherwise.
2443	* @param pVCpu The cross context virtual CPU structure.
2444	* @param idMsr The MSR to check.
2445	*/
2446	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2447	{
2448	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2449	{
2450	switch (idMsr)
2451	{
2452	case MSR_K8_LSTAR:
2453	case MSR_K6_STAR:
2454	case MSR_K8_SF_MASK:
2455	case MSR_K8_KERNEL_GS_BASE:
2456	return true;
2457	}
2458	}
2459	return false;
2460	}
2461
2462
2463	/**
2464	* Loads a set of guests MSRs to allow read/passthru to the guest.
2465	*
2466	* The name of this function is slightly confusing. This function does NOT
2467	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2468	* common prefix for functions dealing with "lazy restoration" of the shared
2469	* MSRs.
2470	*
2471	* @param pVCpu The cross context virtual CPU structure.
2472	*
2473	* @remarks No-long-jump zone!!!
2474	*/
2475	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2476	{
2477	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2478	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2479
2480	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2481	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2482	{
2483	/*
2484	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2485	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2486	* we can skip a few MSR writes.
2487	*
2488	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2489	* guest MSR values in the guest-CPU context might be different to what's currently
2490	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2491	* CPU, see @bugref{8728}.
2492	*/
2493	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2494	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2495	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2496	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2497	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2498	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2499	{
2500	#ifdef VBOX_STRICT
2501	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2502	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2503	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2504	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2505	#endif
2506	}
2507	else
2508	{
2509	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2510	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2511	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2512	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2513	}
2514	}
2515	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2516	}
2517
2518
2519	/**
2520	* Performs lazy restoration of the set of host MSRs if they were previously
2521	* loaded with guest MSR values.
2522	*
2523	* @param pVCpu The cross context virtual CPU structure.
2524	*
2525	* @remarks No-long-jump zone!!!
2526	* @remarks The guest MSRs should have been saved back into the guest-CPU
2527	* context by hmR0VmxImportGuestState()!!!
2528	*/
2529	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2530	{
2531	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2532	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2533
2534	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2535	{
2536	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2537	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2538	{
2539	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2540	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2541	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2542	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2543	}
2544	}
2545	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2546	}
2547
2548
2549	/**
2550	* Verifies that our cached values of the VMCS fields are all consistent with
2551	* what's actually present in the VMCS.
2552	*
2553	* @returns VBox status code.
2554	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2555	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2556	* VMCS content. HMCPU error-field is
2557	* updated, see VMX_VCI_XXX.
2558	* @param pVCpu The cross context virtual CPU structure.
2559	* @param pVmcsInfo The VMCS info. object.
2560	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2561	*/
2562	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2563	{
2564	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2565
2566	uint32_t u32Val;
2567	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2568	AssertRCReturn(rc, rc);
2569	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2570	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2571	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2572	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2573
2574	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2575	AssertRCReturn(rc, rc);
2576	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2577	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2578	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2579	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2580
2581	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2582	AssertRCReturn(rc, rc);
2583	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2584	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2585	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2586	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2587
2588	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2589	AssertRCReturn(rc, rc);
2590	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2591	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2592	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2593	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2594
2595	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2596	{
2597	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2598	AssertRCReturn(rc, rc);
2599	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2600	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2601	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2602	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2603	}
2604
2605	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2606	AssertRCReturn(rc, rc);
2607	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2608	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2609	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2610	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2611
2612	uint64_t u64Val;
2613	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2614	AssertRCReturn(rc, rc);
2615	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2616	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2617	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2618	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2619
2620	NOREF(pcszVmcs);
2621	return VINF_SUCCESS;
2622	}
2623
2624
2625	#ifdef VBOX_STRICT
2626	/**
2627	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2628	*
2629	* @param pVCpu The cross context virtual CPU structure.
2630	* @param pVmcsInfo The VMCS info. object.
2631	*/
2632	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2633	{
2634	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2635
2636	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2637	{
2638	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2639	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2640	uint64_t uVmcsEferMsrVmcs;
2641	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2642	AssertRC(rc);
2643
2644	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2645	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2646	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2647	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2648	}
2649	}
2650
2651
2652	/**
2653	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2654	* VMCS are correct.
2655	*
2656	* @param pVCpu The cross context virtual CPU structure.
2657	* @param pVmcsInfo The VMCS info. object.
2658	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2659	*/
2660	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2661	{
2662	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2663
2664	/* Read the various MSR-area counts from the VMCS. */
2665	uint32_t cEntryLoadMsrs;
2666	uint32_t cExitStoreMsrs;
2667	uint32_t cExitLoadMsrs;
2668	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2669	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2671
2672	/* Verify all the MSR counts are the same. */
2673	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2674	Assert(cExitStoreMsrs == cExitLoadMsrs);
2675	uint32_t const cMsrs = cExitLoadMsrs;
2676
2677	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2678	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2679
2680	/* Verify the MSR counts are within the allocated page size. */
2681	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2682
2683	/* Verify the relevant contents of the MSR areas match. */
2684	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2685	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2686	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2687	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2688	for (uint32_t i = 0; i < cMsrs; i++)
2689	{
2690	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2691	if (fSeparateExitMsrStorePage)
2692	{
2693	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2694	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2695	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2696	}
2697
2698	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2699	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2700	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2701
2702	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2703	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2704	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2705	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2706
2707	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2708	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2709	if (fIsEferMsr)
2710	{
2711	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2712	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2713	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2714	}
2715
2716	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2717	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2718	{
2719	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2720	if (fIsEferMsr)
2721	{
2722	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2723	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2724	}
2725	else
2726	{
2727	if (!fIsNstGstVmcs)
2728	{
2729	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2730	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2731	}
2732	else
2733	{
2734	/*
2735	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2736	* execute a nested-guest with MSR passthrough.
2737	*
2738	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2739	* allow passthrough too.
2740	*/
2741	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2742	Assert(pvMsrBitmapNstGst);
2743	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2744	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2745	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2746	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2747	}
2748	}
2749	}
2750
2751	/* Move to the next MSR. */
2752	pHostMsrLoad++;
2753	pGuestMsrLoad++;
2754	pGuestMsrStore++;
2755	}
2756	}
2757	#endif /* VBOX_STRICT */
2758
2759
2760	/**
2761	* Flushes the TLB using EPT.
2762	*
2763	* @returns VBox status code.
2764	* @param pVCpu The cross context virtual CPU structure of the calling
2765	* EMT. Can be NULL depending on @a enmTlbFlush.
2766	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2767	* enmTlbFlush.
2768	* @param enmTlbFlush Type of flush.
2769	*
2770	* @remarks Caller is responsible for making sure this function is called only
2771	* when NestedPaging is supported and providing @a enmTlbFlush that is
2772	* supported by the CPU.
2773	* @remarks Can be called with interrupts disabled.
2774	*/
2775	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2776	{
2777	uint64_t au64Descriptor[2];
2778	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2779	au64Descriptor[0] = 0;
2780	else
2781	{
2782	Assert(pVCpu);
2783	Assert(pVmcsInfo);
2784	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2785	}
2786	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2787
2788	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2789	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2790
2791	if ( RT_SUCCESS(rc)
2792	&& pVCpu)
2793	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2794	}
2795
2796
2797	/**
2798	* Flushes the TLB using VPID.
2799	*
2800	* @returns VBox status code.
2801	* @param pVCpu The cross context virtual CPU structure of the calling
2802	* EMT. Can be NULL depending on @a enmTlbFlush.
2803	* @param enmTlbFlush Type of flush.
2804	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2805	* on @a enmTlbFlush).
2806	*
2807	* @remarks Can be called with interrupts disabled.
2808	*/
2809	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2810	{
2811	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2812
2813	uint64_t au64Descriptor[2];
2814	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2815	{
2816	au64Descriptor[0] = 0;
2817	au64Descriptor[1] = 0;
2818	}
2819	else
2820	{
2821	AssertPtr(pVCpu);
2822	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2823	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2824	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2825	au64Descriptor[1] = GCPtr;
2826	}
2827
2828	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2829	AssertMsg(rc == VINF_SUCCESS,
2830	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2831
2832	if ( RT_SUCCESS(rc)
2833	&& pVCpu)
2834	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2835	NOREF(rc);
2836	}
2837
2838
2839	/**
2840	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2841	* otherwise there is nothing really to invalidate.
2842	*
2843	* @returns VBox status code.
2844	* @param pVCpu The cross context virtual CPU structure.
2845	* @param GCVirt Guest virtual address of the page to invalidate.
2846	*/
2847	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2848	{
2849	AssertPtr(pVCpu);
2850	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2851
2852	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2853	{
2854	/*
2855	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2856	* the EPT case. See @bugref{6043} and @bugref{6177}.
2857	*
2858	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2859	* as this function maybe called in a loop with individual addresses.
2860	*/
2861	PVM pVM = pVCpu->CTX_SUFF(pVM);
2862	if (pVM->hm.s.vmx.fVpid)
2863	{
2864	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2865	if (fVpidFlush)
2866	{
2867	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2868	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2869	}
2870	else
2871	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2872	}
2873	else if (pVM->hm.s.fNestedPaging)
2874	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2875	}
2876
2877	return VINF_SUCCESS;
2878	}
2879
2880
2881	/**
2882	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2883	* case where neither EPT nor VPID is supported by the CPU.
2884	*
2885	* @param pHostCpu The HM physical-CPU structure.
2886	* @param pVCpu The cross context virtual CPU structure.
2887	*
2888	* @remarks Called with interrupts disabled.
2889	*/
2890	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2891	{
2892	AssertPtr(pVCpu);
2893	AssertPtr(pHostCpu);
2894
2895	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2896
2897	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2898	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2899	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2900	pVCpu->hm.s.fForceTLBFlush = false;
2901	return;
2902	}
2903
2904
2905	/**
2906	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2907	*
2908	* @param pHostCpu The HM physical-CPU structure.
2909	* @param pVCpu The cross context virtual CPU structure.
2910	* @param pVmcsInfo The VMCS info. object.
2911	*
2912	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2913	* nomenclature. The reason is, to avoid confusion in compare statements
2914	* since the host-CPU copies are named "ASID".
2915	*
2916	* @remarks Called with interrupts disabled.
2917	*/
2918	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2919	{
2920	#ifdef VBOX_WITH_STATISTICS
2921	bool fTlbFlushed = false;
2922	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2923	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2924	if (!fTlbFlushed) \
2925	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2926	} while (0)
2927	#else
2928	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2929	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2930	#endif
2931
2932	AssertPtr(pVCpu);
2933	AssertPtr(pHostCpu);
2934	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2935
2936	PVM pVM = pVCpu->CTX_SUFF(pVM);
2937	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2938	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2939	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2940
2941	/*
2942	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2943	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2944	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2945	* cannot reuse the current ASID anymore.
2946	*/
2947	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2948	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2949	{
2950	++pHostCpu->uCurrentAsid;
2951	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2952	{
2953	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2954	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2955	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2956	}
2957
2958	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2959	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2960	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2961
2962	/*
2963	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2964	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2965	*/
2966	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2967	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2968	HMVMX_SET_TAGGED_TLB_FLUSHED();
2969	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2970	}
2971	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2972	{
2973	/*
2974	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2975	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2976	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2977	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2978	* mappings, see @bugref{6568}.
2979	*
2980	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2981	*/
2982	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2983	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2984	HMVMX_SET_TAGGED_TLB_FLUSHED();
2985	}
2986	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
2987	{
2988	/*
2989	* The nested-guest specifies its own guest-physical address to use as the APIC-access
2990	* address which requires flushing the TLB of EPT cached structures.
2991	*
2992	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
2993	*/
2994	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2995	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
2996	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
2997	HMVMX_SET_TAGGED_TLB_FLUSHED();
2998	}
2999
3000
3001	pVCpu->hm.s.fForceTLBFlush = false;
3002	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3003
3004	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3005	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3006	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3007	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3008	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3009	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3010	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3011	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3012	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3013
3014	/* Update VMCS with the VPID. */
3015	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3016	AssertRC(rc);
3017
3018	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3019	}
3020
3021
3022	/**
3023	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3024	*
3025	* @param pHostCpu The HM physical-CPU structure.
3026	* @param pVCpu The cross context virtual CPU structure.
3027	* @param pVmcsInfo The VMCS info. object.
3028	*
3029	* @remarks Called with interrupts disabled.
3030	*/
3031	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3032	{
3033	AssertPtr(pVCpu);
3034	AssertPtr(pHostCpu);
3035	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3036	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3037	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3038
3039	/*
3040	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3041	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3042	*/
3043	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3044	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3045	{
3046	pVCpu->hm.s.fForceTLBFlush = true;
3047	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3048	}
3049
3050	/* Check for explicit TLB flushes. */
3051	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3052	{
3053	pVCpu->hm.s.fForceTLBFlush = true;
3054	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3055	}
3056
3057	/* Check for TLB flushes while switching to/from a nested-guest. */
3058	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3059	{
3060	pVCpu->hm.s.fForceTLBFlush = true;
3061	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3062	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3063	}
3064
3065	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3066	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3067
3068	if (pVCpu->hm.s.fForceTLBFlush)
3069	{
3070	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3071	pVCpu->hm.s.fForceTLBFlush = false;
3072	}
3073	}
3074
3075
3076	/**
3077	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3078	*
3079	* @param pHostCpu The HM physical-CPU structure.
3080	* @param pVCpu The cross context virtual CPU structure.
3081	*
3082	* @remarks Called with interrupts disabled.
3083	*/
3084	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3085	{
3086	AssertPtr(pVCpu);
3087	AssertPtr(pHostCpu);
3088	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3089	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3090	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3091
3092	/*
3093	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3094	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3095	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3096	* cannot reuse the current ASID anymore.
3097	*/
3098	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3099	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3100	{
3101	pVCpu->hm.s.fForceTLBFlush = true;
3102	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3103	}
3104
3105	/* Check for explicit TLB flushes. */
3106	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3107	{
3108	/*
3109	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3110	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3111	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3112	* include fExplicitFlush's too) - an obscure corner case.
3113	*/
3114	pVCpu->hm.s.fForceTLBFlush = true;
3115	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3116	}
3117
3118	/* Check for TLB flushes while switching to/from a nested-guest. */
3119	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3120	{
3121	pVCpu->hm.s.fForceTLBFlush = true;
3122	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3123	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3124	}
3125
3126	PVM pVM = pVCpu->CTX_SUFF(pVM);
3127	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3128	if (pVCpu->hm.s.fForceTLBFlush)
3129	{
3130	++pHostCpu->uCurrentAsid;
3131	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3132	{
3133	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3134	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3135	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3136	}
3137
3138	pVCpu->hm.s.fForceTLBFlush = false;
3139	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3140	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3141	if (pHostCpu->fFlushAsidBeforeUse)
3142	{
3143	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3144	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3145	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3146	{
3147	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3148	pHostCpu->fFlushAsidBeforeUse = false;
3149	}
3150	else
3151	{
3152	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3153	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3154	}
3155	}
3156	}
3157
3158	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3159	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3160	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3161	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3162	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3163	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3164	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3165
3166	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3167	AssertRC(rc);
3168	}
3169
3170
3171	/**
3172	* Flushes the guest TLB entry based on CPU capabilities.
3173	*
3174	* @param pHostCpu The HM physical-CPU structure.
3175	* @param pVCpu The cross context virtual CPU structure.
3176	* @param pVmcsInfo The VMCS info. object.
3177	*
3178	* @remarks Called with interrupts disabled.
3179	*/
3180	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3181	{
3182	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3183	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3184	#endif
3185	PVM pVM = pVCpu->CTX_SUFF(pVM);
3186	switch (pVM->hm.s.vmx.enmTlbFlushType)
3187	{
3188	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3189	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3190	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3191	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3192	default:
3193	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3194	break;
3195	}
3196	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3197	}
3198
3199
3200	/**
3201	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3202	* TLB entries from the host TLB before VM-entry.
3203	*
3204	* @returns VBox status code.
3205	* @param pVM The cross context VM structure.
3206	*/
3207	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3208	{
3209	/*
3210	* Determine optimal flush type for nested paging.
3211	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3212	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3213	*/
3214	if (pVM->hm.s.fNestedPaging)
3215	{
3216	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3217	{
3218	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3219	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3220	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3221	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3222	else
3223	{
3224	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3225	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3226	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3227	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3228	}
3229
3230	/* Make sure the write-back cacheable memory type for EPT is supported. */
3231	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3232	{
3233	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3234	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3235	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3236	}
3237
3238	/* EPT requires a page-walk length of 4. */
3239	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3240	{
3241	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3242	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3243	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3244	}
3245	}
3246	else
3247	{
3248	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3249	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3250	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3251	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3252	}
3253	}
3254
3255	/*
3256	* Determine optimal flush type for VPID.
3257	*/
3258	if (pVM->hm.s.vmx.fVpid)
3259	{
3260	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3261	{
3262	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3263	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3264	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3265	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3266	else
3267	{
3268	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3269	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3270	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3271	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3272	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3273	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3274	pVM->hm.s.vmx.fVpid = false;
3275	}
3276	}
3277	else
3278	{
3279	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3280	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3281	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3282	pVM->hm.s.vmx.fVpid = false;
3283	}
3284	}
3285
3286	/*
3287	* Setup the handler for flushing tagged-TLBs.
3288	*/
3289	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3290	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3291	else if (pVM->hm.s.fNestedPaging)
3292	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3293	else if (pVM->hm.s.vmx.fVpid)
3294	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3295	else
3296	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3297	return VINF_SUCCESS;
3298	}
3299
3300
3301	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3302	/**
3303	* Sets up the shadow VMCS fields arrays.
3304	*
3305	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3306	* executing the guest.
3307	*
3308	* @returns VBox status code.
3309	* @param pVM The cross context VM structure.
3310	*/
3311	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3312	{
3313	/*
3314	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3315	* when the host does not support it.
3316	*/
3317	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3318	if ( !fGstVmwriteAll
3319	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3320	{ /* likely. */ }
3321	else
3322	{
3323	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3324	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3325	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3326	}
3327
3328	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3329	uint32_t cRwFields = 0;
3330	uint32_t cRoFields = 0;
3331	for (uint32_t i = 0; i < cVmcsFields; i++)
3332	{
3333	VMXVMCSFIELD VmcsField;
3334	VmcsField.u = g_aVmcsFields[i];
3335
3336	/*
3337	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3338	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3339	* in the shadow VMCS fields array as they would be redundant.
3340	*
3341	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3342	* we must not include it in the shadow VMCS fields array. Guests attempting to
3343	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3344	* the required behavior.
3345	*/
3346	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3347	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3348	{
3349	/*
3350	* Read-only fields are placed in a separate array so that while syncing shadow
3351	* VMCS fields later (which is more performance critical) we can avoid branches.
3352	*
3353	* However, if the guest can write to all fields (including read-only fields),
3354	* we treat it a as read/write field. Otherwise, writing to these fields would
3355	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3356	*/
3357	if ( fGstVmwriteAll
3358	\|\| !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3359	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3360	else
3361	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3362	}
3363	}
3364
3365	/* Update the counts. */
3366	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3367	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3368	return VINF_SUCCESS;
3369	}
3370
3371
3372	/**
3373	* Sets up the VMREAD and VMWRITE bitmaps.
3374	*
3375	* @param pVM The cross context VM structure.
3376	*/
3377	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3378	{
3379	/*
3380	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3381	*/
3382	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3383	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3384	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3385	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3386	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3387
3388	/*
3389	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3390	* VMREAD and VMWRITE bitmaps.
3391	*/
3392	{
3393	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3394	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3395	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3396	{
3397	uint32_t const uVmcsField = paShadowVmcsFields[i];
3398	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3399	Assert(uVmcsField >> 3 < cbBitmap);
3400	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3401	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3402	}
3403	}
3404
3405	/*
3406	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3407	* if the host supports VMWRITE to all supported VMCS fields.
3408	*/
3409	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3410	{
3411	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3412	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3413	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3414	{
3415	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3416	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3417	Assert(uVmcsField >> 3 < cbBitmap);
3418	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3419	}
3420	}
3421	}
3422	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3423
3424
3425	/**
3426	* Sets up the virtual-APIC page address for the VMCS.
3427	*
3428	* @returns VBox status code.
3429	* @param pVmcsInfo The VMCS info. object.
3430	*/
3431	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
3432	{
3433	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3434	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3435	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3436	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3437	}
3438
3439
3440	/**
3441	* Sets up the MSR-bitmap address for the VMCS.
3442	*
3443	* @returns VBox status code.
3444	* @param pVmcsInfo The VMCS info. object.
3445	*/
3446	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
3447	{
3448	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3449	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3450	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3451	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3452	}
3453
3454
3455	/**
3456	* Sets up the APIC-access page address for the VMCS.
3457	*
3458	* @returns VBox status code.
3459	* @param pVCpu The cross context virtual CPU structure.
3460	*/
3461	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3462	{
3463	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3464	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3465	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3466	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3467	}
3468
3469
3470	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3471	/**
3472	* Sets up the VMREAD bitmap address for the VMCS.
3473	*
3474	* @returns VBox status code.
3475	* @param pVCpu The cross context virtual CPU structure.
3476	*/
3477	DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3478	{
3479	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3480	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3481	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3482	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3483	}
3484
3485
3486	/**
3487	* Sets up the VMWRITE bitmap address for the VMCS.
3488	*
3489	* @returns VBox status code.
3490	* @param pVCpu The cross context virtual CPU structure.
3491	*/
3492	DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3493	{
3494	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3495	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3496	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3497	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3498	}
3499	#endif
3500
3501
3502	/**
3503	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3504	* in the VMCS.
3505	*
3506	* @returns VBox status code.
3507	* @param pVmcsInfo The VMCS info. object.
3508	*/
3509	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
3510	{
3511	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3512	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3513	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3514
3515	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3516	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3517	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3518
3519	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3520	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3521	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3522
3523	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3524	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3525	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3526	AssertRCReturn(rc, rc);
3527	return VINF_SUCCESS;
3528	}
3529
3530
3531	/**
3532	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3533	*
3534	* @param pVCpu The cross context virtual CPU structure.
3535	* @param pVmcsInfo The VMCS info. object.
3536	*/
3537	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3538	{
3539	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3540
3541	/*
3542	* The guest can access the following MSRs (read, write) without causing
3543	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3544	*/
3545	PVM pVM = pVCpu->CTX_SUFF(pVM);
3546	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3547	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3548	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3549	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3550	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3551
3552	/*
3553	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3554	* associated with then. We never need to intercept access (writes need to be
3555	* executed without causing a VM-exit, reads will #GP fault anyway).
3556	*
3557	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3558	* read/write them. We swap the the guest/host MSR value using the
3559	* auto-load/store MSR area.
3560	*/
3561	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3562	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3563	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3564	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3565	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3566	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3567
3568	/*
3569	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3570	* required for 64-bit guests.
3571	*/
3572	if (pVM->hm.s.fAllow64BitGuests)
3573	{
3574	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3575	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3576	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3577	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3578	}
3579
3580	/*
3581	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3582	*/
3583	#ifdef VBOX_STRICT
3584	Assert(pVmcsInfo->pvMsrBitmap);
3585	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3586	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3587	#endif
3588	}
3589
3590
3591	/**
3592	* Sets up pin-based VM-execution controls in the VMCS.
3593	*
3594	* @returns VBox status code.
3595	* @param pVCpu The cross context virtual CPU structure.
3596	* @param pVmcsInfo The VMCS info. object.
3597	*/
3598	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3599	{
3600	PVM pVM = pVCpu->CTX_SUFF(pVM);
3601	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3602	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3603
3604	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3605	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3606
3607	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3608	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3609
3610	/* Enable the VMX-preemption timer. */
3611	if (pVM->hm.s.vmx.fUsePreemptTimer)
3612	{
3613	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3614	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3615	}
3616
3617	#if 0
3618	/* Enable posted-interrupt processing. */
3619	if (pVM->hm.s.fPostedIntrs)
3620	{
3621	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3622	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3623	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3624	}
3625	#endif
3626
3627	if ((fVal & fZap) != fVal)
3628	{
3629	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3630	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3631	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3632	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3633	}
3634
3635	/* Commit it to the VMCS and update our cache. */
3636	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3637	AssertRCReturn(rc, rc);
3638	pVmcsInfo->u32PinCtls = fVal;
3639
3640	return VINF_SUCCESS;
3641	}
3642
3643
3644	/**
3645	* Sets up secondary processor-based VM-execution controls in the VMCS.
3646	*
3647	* @returns VBox status code.
3648	* @param pVCpu The cross context virtual CPU structure.
3649	* @param pVmcsInfo The VMCS info. object.
3650	*/
3651	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3652	{
3653	PVM pVM = pVCpu->CTX_SUFF(pVM);
3654	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3655	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3656
3657	/* WBINVD causes a VM-exit. */
3658	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3659	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3660
3661	/* Enable EPT (aka nested-paging). */
3662	if (pVM->hm.s.fNestedPaging)
3663	fVal \|= VMX_PROC_CTLS2_EPT;
3664
3665	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3666	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3667	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3668	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3669	fVal \|= VMX_PROC_CTLS2_INVPCID;
3670
3671	/* Enable VPID. */
3672	if (pVM->hm.s.vmx.fVpid)
3673	fVal \|= VMX_PROC_CTLS2_VPID;
3674
3675	/* Enable unrestricted guest execution. */
3676	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3677	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3678
3679	#if 0
3680	if (pVM->hm.s.fVirtApicRegs)
3681	{
3682	/* Enable APIC-register virtualization. */
3683	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3684	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3685
3686	/* Enable virtual-interrupt delivery. */
3687	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3688	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3689	}
3690	#endif
3691
3692	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3693	where the TPR shadow resides. */
3694	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3695	* done dynamically. */
3696	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3697	{
3698	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3699	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3700	AssertRCReturn(rc, rc);
3701	}
3702
3703	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3704	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3705	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3706	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3707	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3708
3709	/* Enable Pause-Loop exiting. */
3710	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3711	&& pVM->hm.s.vmx.cPleGapTicks
3712	&& pVM->hm.s.vmx.cPleWindowTicks)
3713	{
3714	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3715
3716	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3717	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3718	AssertRCReturn(rc, rc);
3719	}
3720
3721	if ((fVal & fZap) != fVal)
3722	{
3723	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3724	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3725	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3726	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3727	}
3728
3729	/* Commit it to the VMCS and update our cache. */
3730	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3731	AssertRCReturn(rc, rc);
3732	pVmcsInfo->u32ProcCtls2 = fVal;
3733
3734	return VINF_SUCCESS;
3735	}
3736
3737
3738	/**
3739	* Sets up processor-based VM-execution controls in the VMCS.
3740	*
3741	* @returns VBox status code.
3742	* @param pVCpu The cross context virtual CPU structure.
3743	* @param pVmcsInfo The VMCS info. object.
3744	*/
3745	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3746	{
3747	PVM pVM = pVCpu->CTX_SUFF(pVM);
3748
3749	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3750	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3751
3752	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3753	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3754	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3755	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3756	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3757	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3758	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3759
3760	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3761	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3762	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3763	{
3764	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3765	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3766	}
3767
3768	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3769	if (!pVM->hm.s.fNestedPaging)
3770	{
3771	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3772	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3773	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3774	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3775	}
3776
3777	/* Use TPR shadowing if supported by the CPU. */
3778	if ( PDMHasApic(pVM)
3779	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3780	{
3781	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3782	/* CR8 writes cause a VM-exit based on TPR threshold. */
3783	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3784	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3785	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3786	AssertRCReturn(rc, rc);
3787	}
3788	else
3789	{
3790	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3791	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3792	if (pVM->hm.s.fAllow64BitGuests)
3793	{
3794	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3795	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3796	}
3797	}
3798
3799	/* Use MSR-bitmaps if supported by the CPU. */
3800	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3801	{
3802	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3803	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3804	AssertRCReturn(rc, rc);
3805	}
3806
3807	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3808	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3809	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3810
3811	if ((fVal & fZap) != fVal)
3812	{
3813	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3814	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3815	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3816	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3817	}
3818
3819	/* Commit it to the VMCS and update our cache. */
3820	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3821	AssertRCReturn(rc, rc);
3822	pVmcsInfo->u32ProcCtls = fVal;
3823
3824	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3825	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3826	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3827
3828	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3829	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3830	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3831
3832	/* Sanity check, should not really happen. */
3833	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3834	{ /* likely */ }
3835	else
3836	{
3837	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3838	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3839	}
3840
3841	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3842	return VINF_SUCCESS;
3843	}
3844
3845
3846	/**
3847	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3848	* Processor-based VM-execution) control fields in the VMCS.
3849	*
3850	* @returns VBox status code.
3851	* @param pVCpu The cross context virtual CPU structure.
3852	* @param pVmcsInfo The VMCS info. object.
3853	*/
3854	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3855	{
3856	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3857	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3858	{
3859	int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3860	rc \|= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3861	if (RT_SUCCESS(rc))
3862	{ /* likely */ }
3863	else
3864	{
3865	LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3866	return rc;
3867	}
3868	}
3869	#endif
3870
3871	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3872	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3873	if (RT_SUCCESS(rc))
3874	{
3875	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3876	if (RT_SUCCESS(rc))
3877	{
3878	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3879	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3880	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3881	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3882	if (RT_SUCCESS(rc))
3883	{
3884	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3885	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3886	return VINF_SUCCESS;
3887	}
3888	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3889	}
3890	else
3891	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3892	}
3893	else
3894	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3895	return rc;
3896	}
3897
3898
3899	/**
3900	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3901	*
3902	* We shall setup those exception intercepts that don't change during the
3903	* lifetime of the VM here. The rest are done dynamically while loading the
3904	* guest state.
3905	*
3906	* @returns VBox status code.
3907	* @param pVCpu The cross context virtual CPU structure.
3908	* @param pVmcsInfo The VMCS info. object.
3909	*/
3910	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3911	{
3912	/*
3913	* The following exceptions are always intercepted:
3914	*
3915	* #AC - To prevent the guest from hanging the CPU.
3916	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3917	* recursive #DBs can cause a CPU hang.
3918	* #PF - To sync our shadow page tables when nested-paging is not used.
3919	*/
3920	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3921	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3922	\| RT_BIT(X86_XCPT_DB)
3923	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3924
3925	/* Commit it to the VMCS. */
3926	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3927	AssertRCReturn(rc, rc);
3928
3929	/* Update our cache of the exception bitmap. */
3930	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3931	return VINF_SUCCESS;
3932	}
3933
3934
3935	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3936	/**
3937	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3938	*
3939	* @returns VBox status code.
3940	* @param pVCpu The cross context virtual CPU structure.
3941	* @param pVmcsInfo The VMCS info. object.
3942	*/
3943	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3944	{
3945	PVM pVM = pVCpu->CTX_SUFF(pVM);
3946	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3947	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3948	if (RT_SUCCESS(rc))
3949	{
3950	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3951	if (RT_SUCCESS(rc))
3952	{
3953	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3954	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3955	if (RT_SUCCESS(rc))
3956	return VINF_SUCCESS;
3957	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3958	}
3959	else
3960	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3961	}
3962	else
3963	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3964
3965	return rc;
3966	}
3967	#endif
3968
3969
3970	/**
3971	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3972	* VMX.
3973	*
3974	* @returns VBox status code.
3975	* @param pVCpu The cross context virtual CPU structure.
3976	* @param pVmcsInfo The VMCS info. object.
3977	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3978	*/
3979	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3980	{
3981	Assert(pVmcsInfo->pvVmcs);
3982	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3983
3984	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3985	PVM pVM = pVCpu->CTX_SUFF(pVM);
3986	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3987	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3988
3989	LogFlowFunc(("\n"));
3990
3991	/*
3992	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3993	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3994	*/
3995	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3996	if (RT_SUCCESS(rc))
3997	{
3998	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3999	if (RT_SUCCESS(rc))
4000	{
4001	if (!fIsNstGstVmcs)
4002	{
4003	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
4004	if (RT_SUCCESS(rc))
4005	{
4006	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4007	if (RT_SUCCESS(rc))
4008	{
4009	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4010	if (RT_SUCCESS(rc))
4011	{
4012	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4013	if (RT_SUCCESS(rc))
4014	{
4015	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4016	/*
4017	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4018	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4019	* making it fit for use when VMCS shadowing is later enabled.
4020	*/
4021	if (pVmcsInfo->pvShadowVmcs)
4022	{
4023	VMXVMCSREVID VmcsRevId;
4024	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4025	VmcsRevId.n.fIsShadowVmcs = 1;
4026	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4027	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4028	if (RT_SUCCESS(rc))
4029	{ /* likely */ }
4030	else
4031	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4032	}
4033	#endif
4034	}
4035	else
4036	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4037	}
4038	else
4039	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4040	}
4041	else
4042	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4043	}
4044	else
4045	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4046	}
4047	else
4048	{
4049	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4050	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4051	if (RT_SUCCESS(rc))
4052	{ /* likely */ }
4053	else
4054	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4055	#else
4056	AssertFailed();
4057	#endif
4058	}
4059	}
4060	else
4061	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4062	}
4063	else
4064	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4065
4066	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4067	if (RT_SUCCESS(rc))
4068	{
4069	rc = hmR0VmxClearVmcs(pVmcsInfo);
4070	if (RT_SUCCESS(rc))
4071	{ /* likely */ }
4072	else
4073	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4074	}
4075
4076	/*
4077	* Update the last-error record both for failures and success, so we
4078	* can propagate the status code back to ring-3 for diagnostics.
4079	*/
4080	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4081	NOREF(pszVmcs);
4082	return rc;
4083	}
4084
4085
4086	/**
4087	* Does global VT-x initialization (called during module initialization).
4088	*
4089	* @returns VBox status code.
4090	*/
4091	VMMR0DECL(int) VMXR0GlobalInit(void)
4092	{
4093	#ifdef HMVMX_USE_FUNCTION_TABLE
4094	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4095	# ifdef VBOX_STRICT
4096	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4097	Assert(g_apfnVMExitHandlers[i]);
4098	# endif
4099	#endif
4100	return VINF_SUCCESS;
4101	}
4102
4103
4104	/**
4105	* Does global VT-x termination (called during module termination).
4106	*/
4107	VMMR0DECL(void) VMXR0GlobalTerm()
4108	{
4109	/* Nothing to do currently. */
4110	}
4111
4112
4113	/**
4114	* Sets up and activates VT-x on the current CPU.
4115	*
4116	* @returns VBox status code.
4117	* @param pHostCpu The HM physical-CPU structure.
4118	* @param pVM The cross context VM structure. Can be
4119	* NULL after a host resume operation.
4120	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4121	* fEnabledByHost is @c true).
4122	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4123	* @a fEnabledByHost is @c true).
4124	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4125	* enable VT-x on the host.
4126	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4127	*/
4128	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4129	PCSUPHWVIRTMSRS pHwvirtMsrs)
4130	{
4131	AssertPtr(pHostCpu);
4132	AssertPtr(pHwvirtMsrs);
4133	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4134
4135	/* Enable VT-x if it's not already enabled by the host. */
4136	if (!fEnabledByHost)
4137	{
4138	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4139	if (RT_FAILURE(rc))
4140	return rc;
4141	}
4142
4143	/*
4144	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4145	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4146	* invalidated when flushing by VPID.
4147	*/
4148	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4149	{
4150	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4151	pHostCpu->fFlushAsidBeforeUse = false;
4152	}
4153	else
4154	pHostCpu->fFlushAsidBeforeUse = true;
4155
4156	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4157	++pHostCpu->cTlbFlushes;
4158
4159	return VINF_SUCCESS;
4160	}
4161
4162
4163	/**
4164	* Deactivates VT-x on the current CPU.
4165	*
4166	* @returns VBox status code.
4167	* @param pvCpuPage Pointer to the VMXON region.
4168	* @param HCPhysCpuPage Physical address of the VMXON region.
4169	*
4170	* @remarks This function should never be called when SUPR0EnableVTx() or
4171	* similar was used to enable VT-x on the host.
4172	*/
4173	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4174	{
4175	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4176
4177	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4178	return hmR0VmxLeaveRootMode();
4179	}
4180
4181
4182	/**
4183	* Does per-VM VT-x initialization.
4184	*
4185	* @returns VBox status code.
4186	* @param pVM The cross context VM structure.
4187	*/
4188	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4189	{
4190	AssertPtr(pVM);
4191	LogFlowFunc(("pVM=%p\n", pVM));
4192
4193	int rc = hmR0VmxStructsAlloc(pVM);
4194	if (RT_FAILURE(rc))
4195	{
4196	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4197	return rc;
4198	}
4199
4200	return VINF_SUCCESS;
4201	}
4202
4203
4204	/**
4205	* Does per-VM VT-x termination.
4206	*
4207	* @returns VBox status code.
4208	* @param pVM The cross context VM structure.
4209	*/
4210	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4211	{
4212	AssertPtr(pVM);
4213	LogFlowFunc(("pVM=%p\n", pVM));
4214
4215	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4216	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4217	{
4218	Assert(pVM->hm.s.vmx.pvScratch);
4219	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4220	}
4221	#endif
4222	hmR0VmxStructsFree(pVM);
4223	return VINF_SUCCESS;
4224	}
4225
4226
4227	/**
4228	* Sets up the VM for execution using hardware-assisted VMX.
4229	* This function is only called once per-VM during initialization.
4230	*
4231	* @returns VBox status code.
4232	* @param pVM The cross context VM structure.
4233	*/
4234	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4235	{
4236	AssertPtr(pVM);
4237	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4238
4239	LogFlowFunc(("pVM=%p\n", pVM));
4240
4241	/*
4242	* At least verify if VMX is enabled, since we can't check if we're in
4243	* VMX root mode or not without causing a #GP.
4244	*/
4245	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4246	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4247	{ /* likely */ }
4248	else
4249	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4250
4251	/*
4252	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4253	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4254	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4255	*/
4256	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4257	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4258	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4259	{
4260	LogRelFunc(("Invalid real-on-v86 state.\n"));
4261	return VERR_INTERNAL_ERROR;
4262	}
4263
4264	/* Initialize these always, see hmR3InitFinalizeR0().*/
4265	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4266	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4267
4268	/* Setup the tagged-TLB flush handlers. */
4269	int rc = hmR0VmxSetupTaggedTlb(pVM);
4270	if (RT_FAILURE(rc))
4271	{
4272	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4273	return rc;
4274	}
4275
4276	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4277	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4278	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4279	{
4280	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4281	if (RT_SUCCESS(rc))
4282	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4283	else
4284	{
4285	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4286	return rc;
4287	}
4288	}
4289	#endif
4290
4291	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4292	{
4293	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4294	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4295
4296	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4297	if (RT_SUCCESS(rc))
4298	{
4299	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4300	if (pVM->cpum.ro.GuestFeatures.fVmx)
4301	{
4302	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4303	if (RT_SUCCESS(rc))
4304	{ /* likely */ }
4305	else
4306	{
4307	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4308	return rc;
4309	}
4310	}
4311	#endif
4312	}
4313	else
4314	{
4315	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4316	return rc;
4317	}
4318	}
4319
4320	return VINF_SUCCESS;
4321	}
4322
4323
4324	/**
4325	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4326	* the VMCS.
4327	*
4328	* @returns VBox status code.
4329	*/
4330	static int hmR0VmxExportHostControlRegs(void)
4331	{
4332	RTCCUINTREG uReg = ASMGetCR0();
4333	int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, uReg);
4334	AssertRCReturn(rc, rc);
4335
4336	uReg = ASMGetCR3();
4337	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, uReg);
4338	AssertRCReturn(rc, rc);
4339
4340	uReg = ASMGetCR4();
4341	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, uReg);
4342	AssertRCReturn(rc, rc);
4343	return rc;
4344	}
4345
4346
4347	/**
4348	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4349	* the host-state area in the VMCS.
4350	*
4351	* @returns VBox status code.
4352	* @param pVCpu The cross context virtual CPU structure.
4353	*/
4354	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4355	{
4356	/**
4357	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4358	* requirements. See hmR0VmxExportHostSegmentRegs().
4359	*/
4360	#define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4361	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4362	{ \
4363	bool fValidSelector = true; \
4364	if ((selValue) & X86_SEL_LDT) \
4365	{ \
4366	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4367	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4368	} \
4369	if (fValidSelector) \
4370	{ \
4371	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4372	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4373	} \
4374	(selValue) = 0; \
4375	}
4376
4377	/*
4378	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4379	* will be messed up. We should -not- save the messed up state without restoring
4380	* the original host-state, see @bugref{7240}.
4381	*
4382	* This apparently can happen (most likely the FPU changes), deal with it rather than
4383	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4384	*/
4385	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4386	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4387	{
4388	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4389	pVCpu->idCpu));
4390	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4391	}
4392	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4393
4394	/*
4395	* Host segment registers.
4396	*/
4397	RTSEL uSelES = ASMGetES();
4398	RTSEL uSelCS = ASMGetCS();
4399	RTSEL uSelSS = ASMGetSS();
4400	RTSEL uSelDS = ASMGetDS();
4401	RTSEL uSelFS = ASMGetFS();
4402	RTSEL uSelGS = ASMGetGS();
4403	RTSEL uSelTR = ASMGetTR();
4404
4405	/*
4406	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4407	* gain VM-entry and restore them before we get preempted.
4408	*
4409	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4410	*/
4411	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4412	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4413	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4414	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4415
4416	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4417	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4418	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4419	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4420	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4421	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4422	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4423	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4424	Assert(uSelCS);
4425	Assert(uSelTR);
4426
4427	/* Write these host selector fields into the host-state area in the VMCS. */
4428	int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4429	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4430	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4431	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES);
4432	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4433	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4434	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4435	AssertRCReturn(rc, rc);
4436
4437	/*
4438	* Host GDTR and IDTR.
4439	*/
4440	RTGDTR Gdtr;
4441	RTIDTR Idtr;
4442	RT_ZERO(Gdtr);
4443	RT_ZERO(Idtr);
4444	ASMGetGDTR(&Gdtr);
4445	ASMGetIDTR(&Idtr);
4446	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4447	rc \|= VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4448	AssertRCReturn(rc, rc);
4449
4450	/*
4451	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4452	* them to the maximum limit (0xffff) on every VM-exit.
4453	*/
4454	if (Gdtr.cbGdt != 0xffff)
4455	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4456
4457	/*
4458	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4459	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4460	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4461	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4462	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4463	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4464	* at 0xffff on hosts where we are sure it won't cause trouble.
4465	*/
4466	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4467	if (Idtr.cbIdt < 0x0fff)
4468	#else
4469	if (Idtr.cbIdt != 0xffff)
4470	#endif
4471	{
4472	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4473	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4474	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4475	}
4476
4477	/*
4478	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4479	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4480	* RPL should be too in most cases.
4481	*/
4482	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4483	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4484
4485	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4486	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4487
4488	/*
4489	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4490	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4491	* restoration if the host has something else. Task switching is not supported in 64-bit
4492	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4493	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4494	*
4495	* [1] See Intel spec. 3.5 "System Descriptor Types".
4496	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4497	*/
4498	PVM pVM = pVCpu->CTX_SUFF(pVM);
4499	Assert(pDesc->System.u4Type == 11);
4500	if ( pDesc->System.u16LimitLow != 0x67
4501	\|\| pDesc->System.u4LimitHigh)
4502	{
4503	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4504	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4505	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4506	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4507	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4508	}
4509
4510	/*
4511	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4512	*/
4513	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4514	{
4515	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4516	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4517	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4518	{
4519	/* The GDT is read-only but the writable GDT is available. */
4520	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4521	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4522	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4523	AssertRCReturn(rc, rc);
4524	}
4525	}
4526
4527	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase);
4528	AssertRCReturn(rc, rc);
4529
4530	/*
4531	* Host FS base and GS base.
4532	*/
4533	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4534	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4535	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4536	rc \|= VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4537	AssertRCReturn(rc, rc);
4538
4539	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4540	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4541	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4542	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4543	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4544
4545	return VINF_SUCCESS;
4546	#undef VMXLOCAL_ADJUST_HOST_SEG
4547	}
4548
4549
4550	/**
4551	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4552	* host-state area of the VMCS.
4553	*
4554	* These MSRs will be automatically restored on the host after every successful
4555	* VM-exit.
4556	*
4557	* @returns VBox status code.
4558	* @param pVCpu The cross context virtual CPU structure.
4559	*
4560	* @remarks No-long-jump zone!!!
4561	*/
4562	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4563	{
4564	AssertPtr(pVCpu);
4565
4566	/*
4567	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4568	* rather than swapping them on every VM-entry.
4569	*/
4570	hmR0VmxLazySaveHostMsrs(pVCpu);
4571
4572	/*
4573	* Host Sysenter MSRs.
4574	*/
4575	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4576	rc \|= VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4577	rc \|= VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4578	AssertRCReturn(rc, rc);
4579
4580	/*
4581	* Host EFER MSR.
4582	*
4583	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4584	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4585	*/
4586	PVM pVM = pVCpu->CTX_SUFF(pVM);
4587	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4588	{
4589	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4590	AssertRCReturn(rc, rc);
4591	}
4592
4593	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4594	* hmR0VmxExportGuestEntryExitCtls(). */
4595
4596	return VINF_SUCCESS;
4597	}
4598
4599
4600	/**
4601	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4602	*
4603	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4604	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4605	*
4606	* @returns true if we need to load guest EFER, false otherwise.
4607	* @param pVCpu The cross context virtual CPU structure.
4608	*
4609	* @remarks Requires EFER, CR4.
4610	* @remarks No-long-jump zone!!!
4611	*/
4612	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4613	{
4614	#ifdef HMVMX_ALWAYS_SWAP_EFER
4615	RT_NOREF(pVCpu);
4616	return true;
4617	#else
4618	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4619	PVM pVM = pVCpu->CTX_SUFF(pVM);
4620	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4621	uint64_t const u64GuestEfer = pCtx->msrEFER;
4622
4623	/*
4624	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4625	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4626	*/
4627	if ( CPUMIsGuestInLongModeEx(pCtx)
4628	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4629	return true;
4630
4631	/*
4632	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4633	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4634	*
4635	* See Intel spec. 4.5 "IA-32e Paging".
4636	* See Intel spec. 4.1.1 "Three Paging Modes".
4637	*
4638	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4639	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4640	*/
4641	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4642	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4643	if ( (pCtx->cr4 & X86_CR4_PAE)
4644	&& (pCtx->cr0 & X86_CR0_PG)
4645	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4646	{
4647	/* Assert that host is NX capable. */
4648	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4649	return true;
4650	}
4651
4652	return false;
4653	#endif
4654	}
4655
4656	/**
4657	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4658	* VMCS.
4659	*
4660	* This is typically required when the guest changes paging mode.
4661	*
4662	* @returns VBox status code.
4663	* @param pVCpu The cross context virtual CPU structure.
4664	* @param pVmxTransient The VMX-transient structure.
4665	*
4666	* @remarks Requires EFER.
4667	* @remarks No-long-jump zone!!!
4668	*/
4669	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4670	{
4671	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4672	{
4673	PVM pVM = pVCpu->CTX_SUFF(pVM);
4674	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4675
4676	/*
4677	* VM-entry controls.
4678	*/
4679	{
4680	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4681	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4682
4683	/*
4684	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4685	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4686	*
4687	* For nested-guests, this is a mandatory VM-entry control. It's also
4688	* required because we do not want to leak host bits to the nested-guest.
4689	*/
4690	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4691
4692	/*
4693	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4694	*
4695	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4696	* required to get the nested-guest working with hardware-assisted VMX execution.
4697	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4698	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4699	* here rather than while merging the guest VMCS controls.
4700	*/
4701	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4702	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4703	else
4704	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4705
4706	/*
4707	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4708	*
4709	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4710	* regardless of whether the nested-guest VMCS specifies it because we are free to
4711	* load whatever MSRs we require and we do not need to modify the guest visible copy
4712	* of the VM-entry MSR load area.
4713	*/
4714	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4715	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4716	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4717	else
4718	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4719
4720	/*
4721	* The following should -not- be set (since we're not in SMM mode):
4722	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4723	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4724	*/
4725
4726	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4727	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4728
4729	if ((fVal & fZap) == fVal)
4730	{ /* likely */ }
4731	else
4732	{
4733	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4734	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4735	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4736	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4737	}
4738
4739	/* Commit it to the VMCS. */
4740	if (pVmcsInfo->u32EntryCtls != fVal)
4741	{
4742	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4743	AssertRCReturn(rc, rc);
4744	pVmcsInfo->u32EntryCtls = fVal;
4745	}
4746	}
4747
4748	/*
4749	* VM-exit controls.
4750	*/
4751	{
4752	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4753	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4754
4755	/*
4756	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4757	* supported the 1-setting of this bit.
4758	*
4759	* For nested-guests, we set the "save debug controls" as the converse
4760	* "load debug controls" is mandatory for nested-guests anyway.
4761	*/
4762	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4763
4764	/*
4765	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4766	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4767	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4768	* hmR0VmxExportHostMsrs().
4769	*
4770	* For nested-guests, we always set this bit as we do not support 32-bit
4771	* hosts.
4772	*/
4773	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4774
4775	/*
4776	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4777	*
4778	* For nested-guests, we should use the "save IA32_EFER" control if we also
4779	* used the "load IA32_EFER" control while exporting VM-entry controls.
4780	*/
4781	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4782	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4783	{
4784	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4785	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4786	}
4787
4788	/*
4789	* Enable saving of the VMX-preemption timer value on VM-exit.
4790	* For nested-guests, currently not exposed/used.
4791	*/
4792	if ( pVM->hm.s.vmx.fUsePreemptTimer
4793	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4794	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4795
4796	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4797	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4798
4799	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4800	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4801	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4802
4803	if ((fVal & fZap) == fVal)
4804	{ /* likely */ }
4805	else
4806	{
4807	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4808	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4809	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4810	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4811	}
4812
4813	/* Commit it to the VMCS. */
4814	if (pVmcsInfo->u32ExitCtls != fVal)
4815	{
4816	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4817	AssertRCReturn(rc, rc);
4818	pVmcsInfo->u32ExitCtls = fVal;
4819	}
4820	}
4821
4822	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4823	}
4824	return VINF_SUCCESS;
4825	}
4826
4827
4828	/**
4829	* Sets the TPR threshold in the VMCS.
4830	*
4831	* @returns VBox status code.
4832	* @param pVCpu The cross context virtual CPU structure.
4833	* @param pVmcsInfo The VMCS info. object.
4834	* @param u32TprThreshold The TPR threshold (task-priority class only).
4835	*/
4836	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4837	{
4838	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4839	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4840	RT_NOREF2(pVCpu, pVmcsInfo);
4841	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4842	}
4843
4844
4845	/**
4846	* Exports the guest APIC TPR state into the VMCS.
4847	*
4848	* @returns VBox status code.
4849	* @param pVCpu The cross context virtual CPU structure.
4850	* @param pVmxTransient The VMX-transient structure.
4851	*
4852	* @remarks No-long-jump zone!!!
4853	*/
4854	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4855	{
4856	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4857	{
4858	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4859
4860	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4861	if (!pVmxTransient->fIsNestedGuest)
4862	{
4863	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4864	&& APICIsEnabled(pVCpu))
4865	{
4866	/*
4867	* Setup TPR shadowing.
4868	*/
4869	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4870	{
4871	bool fPendingIntr = false;
4872	uint8_t u8Tpr = 0;
4873	uint8_t u8PendingIntr = 0;
4874	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4875	AssertRCReturn(rc, rc);
4876
4877	/*
4878	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4879	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4880	* priority of the pending interrupt so we can deliver the interrupt. If there
4881	* are no interrupts pending, set threshold to 0 to not cause any
4882	* TPR-below-threshold VM-exits.
4883	*/
4884	uint32_t u32TprThreshold = 0;
4885	if (fPendingIntr)
4886	{
4887	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4888	(which is the Task-Priority Class). */
4889	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4890	const uint8_t u8TprPriority = u8Tpr >> 4;
4891	if (u8PendingPriority <= u8TprPriority)
4892	u32TprThreshold = u8PendingPriority;
4893	}
4894
4895	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4896	AssertRCReturn(rc, rc);
4897	}
4898	}
4899	}
4900	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4901	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4902	}
4903	return VINF_SUCCESS;
4904	}
4905
4906
4907	/**
4908	* Gets the guest interruptibility-state.
4909	*
4910	* @returns Guest's interruptibility-state.
4911	* @param pVCpu The cross context virtual CPU structure.
4912	* @param pVmxTransient The VMX-transient structure.
4913	*
4914	* @remarks No-long-jump zone!!!
4915	*/
4916	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4917	{
4918	/*
4919	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4920	*/
4921	uint32_t fIntrState = 0;
4922	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4923	{
4924	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
4925	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
4926
4927	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4928	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4929	{
4930	if (pCtx->eflags.Bits.u1IF)
4931	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4932	else
4933	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4934	}
4935	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4936	{
4937	/*
4938	* We can clear the inhibit force flag as even if we go back to the recompiler
4939	* without executing guest code in VT-x, the flag's condition to be cleared is
4940	* met and thus the cleared state is correct.
4941	*/
4942	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4943	}
4944	}
4945
4946	/*
4947	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4948	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4949	* setting this would block host-NMIs and IRET will not clear the blocking.
4950	*
4951	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4952	*
4953	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4954	*/
4955	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4956	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4957	&& CPUMIsGuestNmiBlocking(pVCpu))
4958	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4959
4960	return fIntrState;
4961	}
4962
4963
4964	/**
4965	* Exports the exception intercepts required for guest execution in the VMCS.
4966	*
4967	* @returns VBox status code.
4968	* @param pVCpu The cross context virtual CPU structure.
4969	* @param pVmxTransient The VMX-transient structure.
4970	*
4971	* @remarks No-long-jump zone!!!
4972	*/
4973	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4974	{
4975	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4976	{
4977	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4978	if ( !pVmxTransient->fIsNestedGuest
4979	&& pVCpu->hm.s.fGIMTrapXcptUD)
4980	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4981	else
4982	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4983
4984	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4985	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4986	}
4987	return VINF_SUCCESS;
4988	}
4989
4990
4991	/**
4992	* Exports the guest's RIP into the guest-state area in the VMCS.
4993	*
4994	* @returns VBox status code.
4995	* @param pVCpu The cross context virtual CPU structure.
4996	*
4997	* @remarks No-long-jump zone!!!
4998	*/
4999	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5000	{
5001	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5002	{
5003	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5004
5005	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5006	AssertRCReturn(rc, rc);
5007
5008	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5009	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5010	}
5011	return VINF_SUCCESS;
5012	}
5013
5014
5015	/**
5016	* Exports the guest's RSP into the guest-state area in the VMCS.
5017	*
5018	* @returns VBox status code.
5019	* @param pVCpu The cross context virtual CPU structure.
5020	*
5021	* @remarks No-long-jump zone!!!
5022	*/
5023	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5024	{
5025	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5026	{
5027	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5028
5029	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5030	AssertRCReturn(rc, rc);
5031
5032	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5033	}
5034	return VINF_SUCCESS;
5035	}
5036
5037
5038	/**
5039	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5040	*
5041	* @returns VBox status code.
5042	* @param pVCpu The cross context virtual CPU structure.
5043	* @param pVmxTransient The VMX-transient structure.
5044	*
5045	* @remarks No-long-jump zone!!!
5046	*/
5047	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5048	{
5049	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5050	{
5051	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5052
5053	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5054	Let us assert it as such and use 32-bit VMWRITE. */
5055	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5056	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5057	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5058	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5059
5060	/*
5061	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5062	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5063	* can run the real-mode guest code under Virtual 8086 mode.
5064	*/
5065	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5066	if (pVmcsInfo->RealMode.fRealOnV86Active)
5067	{
5068	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5069	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5070	Assert(!pVmxTransient->fIsNestedGuest);
5071	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5072	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5073	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5074	}
5075
5076	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5077	AssertRCReturn(rc, rc);
5078
5079	/*
5080	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5081	*
5082	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5083	* through the hypervisor debugger using EFLAGS.TF.
5084	*/
5085	if ( !pVmxTransient->fIsNestedGuest
5086	&& !pVCpu->hm.s.fSingleInstruction
5087	&& fEFlags.Bits.u1TF)
5088	{
5089	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5090	* premature trips to ring-3 esp since IEM does not yet handle it. */
5091	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5092	AssertRCReturn(rc, rc);
5093	}
5094	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5095	* nested-guest VMCS. */
5096
5097	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5098	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5099	}
5100	return VINF_SUCCESS;
5101	}
5102
5103
5104	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5105	/**
5106	* Copies the nested-guest VMCS to the shadow VMCS.
5107	*
5108	* @returns VBox status code.
5109	* @param pVCpu The cross context virtual CPU structure.
5110	* @param pVmcsInfo The VMCS info. object.
5111	*
5112	* @remarks No-long-jump zone!!!
5113	*/
5114	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5115	{
5116	PVM pVM = pVCpu->CTX_SUFF(pVM);
5117	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5118
5119	/*
5120	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5121	* current VMCS, as we may try saving guest lazy MSRs.
5122	*
5123	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5124	* calling the import VMCS code which is currently performing the guest MSR reads
5125	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5126	* and the rest of the VMX leave session machinery.
5127	*/
5128	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5129
5130	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5131	if (RT_SUCCESS(rc))
5132	{
5133	/*
5134	* Copy all guest read/write VMCS fields.
5135	*
5136	* We don't check for VMWRITE failures here for performance reasons and
5137	* because they are not expected to fail, barring irrecoverable conditions
5138	* like hardware errors.
5139	*/
5140	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5141	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5142	{
5143	uint64_t u64Val;
5144	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5145	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5146	VMXWriteVmcs64(uVmcsField, u64Val);
5147	}
5148
5149	/*
5150	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5151	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5152	*/
5153	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5154	{
5155	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5156	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5157	{
5158	uint64_t u64Val;
5159	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5160	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5161	VMXWriteVmcs64(uVmcsField, u64Val);
5162	}
5163	}
5164
5165	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5166	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5167	}
5168
5169	ASMSetFlags(fEFlags);
5170	return rc;
5171	}
5172
5173
5174	/**
5175	* Copies the shadow VMCS to the nested-guest VMCS.
5176	*
5177	* @returns VBox status code.
5178	* @param pVCpu The cross context virtual CPU structure.
5179	* @param pVmcsInfo The VMCS info. object.
5180	*
5181	* @remarks Called with interrupts disabled.
5182	*/
5183	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5184	{
5185	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5186	PVM pVM = pVCpu->CTX_SUFF(pVM);
5187	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5188
5189	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5190	if (RT_SUCCESS(rc))
5191	{
5192	/*
5193	* Copy guest read/write fields from the shadow VMCS.
5194	* Guest read-only fields cannot be modified, so no need to copy them.
5195	*
5196	* We don't check for VMREAD failures here for performance reasons and
5197	* because they are not expected to fail, barring irrecoverable conditions
5198	* like hardware errors.
5199	*/
5200	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5201	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5202	{
5203	uint64_t u64Val;
5204	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5205	VMXReadVmcs64(uVmcsField, &u64Val);
5206	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5207	}
5208
5209	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5210	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5211	}
5212	return rc;
5213	}
5214
5215
5216	/**
5217	* Enables VMCS shadowing for the given VMCS info. object.
5218	*
5219	* @param pVmcsInfo The VMCS info. object.
5220	*
5221	* @remarks No-long-jump zone!!!
5222	*/
5223	static void hmR0VmxEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5224	{
5225	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5226	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5227	{
5228	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5229	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5230	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5231	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5232	AssertRC(rc);
5233
5234	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5235	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5236	Log4Func(("Enabled\n"));
5237	}
5238	}
5239
5240
5241	/**
5242	* Disables VMCS shadowing for the given VMCS info. object.
5243	*
5244	* @param pVmcsInfo The VMCS info. object.
5245	*
5246	* @remarks No-long-jump zone!!!
5247	*/
5248	static void hmR0VmxDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5249	{
5250	/*
5251	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5252	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5253	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5254	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5255	*
5256	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5257	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5258	*/
5259	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5260	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5261	{
5262	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5263	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5264	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5265	AssertRC(rc);
5266
5267	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5268	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5269	Log4Func(("Disabled\n"));
5270	}
5271	}
5272	#endif
5273
5274
5275	/**
5276	* Exports the guest hardware-virtualization state.
5277	*
5278	* @returns VBox status code.
5279	* @param pVCpu The cross context virtual CPU structure.
5280	* @param pVmxTransient The VMX-transient structure.
5281	*
5282	* @remarks No-long-jump zone!!!
5283	*/
5284	static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5285	{
5286	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5287	{
5288	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5289	/*
5290	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5291	* VMCS shadowing.
5292	*/
5293	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5294	{
5295	/*
5296	* If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5297	* copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5298	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5299	*
5300	* We check for VMX root mode here in case the guest executes VMXOFF without
5301	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5302	* not clear the current VMCS pointer.
5303	*/
5304	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5305	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5306	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5307	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5308	{
5309	/* Paranoia. */
5310	Assert(!pVmxTransient->fIsNestedGuest);
5311
5312	/*
5313	* For performance reasons, also check if the guest hypervisor's current VMCS
5314	* was newly loaded or modified before copying it to the shadow VMCS.
5315	*/
5316	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5317	{
5318	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5319	AssertRCReturn(rc, rc);
5320	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5321	}
5322	hmR0VmxEnableVmcsShadowing(pVmcsInfo);
5323	}
5324	else
5325	hmR0VmxDisableVmcsShadowing(pVmcsInfo);
5326	}
5327	#else
5328	NOREF(pVmxTransient);
5329	#endif
5330	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5331	}
5332	return VINF_SUCCESS;
5333	}
5334
5335
5336	/**
5337	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5338	*
5339	* The guest FPU state is always pre-loaded hence we don't need to bother about
5340	* sharing FPU related CR0 bits between the guest and host.
5341	*
5342	* @returns VBox status code.
5343	* @param pVCpu The cross context virtual CPU structure.
5344	* @param pVmxTransient The VMX-transient structure.
5345	*
5346	* @remarks No-long-jump zone!!!
5347	*/
5348	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5349	{
5350	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5351	{
5352	PVM pVM = pVCpu->CTX_SUFF(pVM);
5353	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5354
5355	/*
5356	* Figure out fixed CR0 bits in VMX operation.
5357	*/
5358	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5359	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5360	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5361	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5362	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5363	else
5364	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5365
5366	if (!pVmxTransient->fIsNestedGuest)
5367	{
5368	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5369	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5370	uint64_t const u64ShadowCr0 = u64GuestCr0;
5371	Assert(!RT_HI_U32(u64GuestCr0));
5372
5373	/*
5374	* Setup VT-x's view of the guest CR0.
5375	*/
5376	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5377	if (pVM->hm.s.fNestedPaging)
5378	{
5379	if (CPUMIsGuestPagingEnabled(pVCpu))
5380	{
5381	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5382	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5383	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5384	}
5385	else
5386	{
5387	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5388	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5389	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5390	}
5391
5392	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5393	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5394	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5395	}
5396	else
5397	{
5398	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5399	u64GuestCr0 \|= X86_CR0_WP;
5400	}
5401
5402	/*
5403	* Guest FPU bits.
5404	*
5405	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5406	* using CR0.TS.
5407	*
5408	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5409	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5410	*/
5411	u64GuestCr0 \|= X86_CR0_NE;
5412
5413	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5414	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5415
5416	/*
5417	* Update exception intercepts.
5418	*/
5419	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5420	if (pVmcsInfo->RealMode.fRealOnV86Active)
5421	{
5422	Assert(PDMVmmDevHeapIsEnabled(pVM));
5423	Assert(pVM->hm.s.vmx.pRealModeTSS);
5424	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5425	}
5426	else
5427	{
5428	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5429	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5430	if (fInterceptMF)
5431	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5432	}
5433
5434	/* Additional intercepts for debugging, define these yourself explicitly. */
5435	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5436	uXcptBitmap \|= 0
5437	\| RT_BIT(X86_XCPT_BP)
5438	\| RT_BIT(X86_XCPT_DE)
5439	\| RT_BIT(X86_XCPT_NM)
5440	\| RT_BIT(X86_XCPT_TS)
5441	\| RT_BIT(X86_XCPT_UD)
5442	\| RT_BIT(X86_XCPT_NP)
5443	\| RT_BIT(X86_XCPT_SS)
5444	\| RT_BIT(X86_XCPT_GP)
5445	\| RT_BIT(X86_XCPT_PF)
5446	\| RT_BIT(X86_XCPT_MF)
5447	;
5448	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5449	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5450	#endif
5451	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5452	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5453	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5454
5455	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5456	u64GuestCr0 \|= fSetCr0;
5457	u64GuestCr0 &= fZapCr0;
5458	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5459
5460	/* Commit the CR0 and related fields to the guest VMCS. */
5461	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5462	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5463	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5464	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5465	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5466	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5467	AssertRCReturn(rc, rc);
5468
5469	/* Update our caches. */
5470	pVmcsInfo->u32ProcCtls = uProcCtls;
5471	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5472
5473	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5474	}
5475	else
5476	{
5477	/*
5478	* With nested-guests, we may have extended the guest/host mask here since we
5479	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5480	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5481	* originally supplied. We must copy those bits from the nested-guest CR0 into
5482	* the nested-guest CR0 read-shadow.
5483	*/
5484	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5485	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5486	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5487	Assert(!RT_HI_U32(u64GuestCr0));
5488	Assert(u64GuestCr0 & X86_CR0_NE);
5489
5490	/*
5491	* Apply the hardware specified fixed CR0 bits and enable caching.
5492	* Note! We could be altering our VMX emulation's fixed bits. We thus
5493	* need to re-apply them while importing CR0.
5494	*/
5495	u64GuestCr0 \|= fSetCr0;
5496	u64GuestCr0 &= fZapCr0;
5497	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5498
5499	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5500	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5501	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5502	AssertRCReturn(rc, rc);
5503
5504	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5505	}
5506
5507	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5508	}
5509
5510	return VINF_SUCCESS;
5511	}
5512
5513
5514	/**
5515	* Exports the guest control registers (CR3, CR4) into the guest-state area
5516	* in the VMCS.
5517	*
5518	* @returns VBox strict status code.
5519	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5520	* without unrestricted guest access and the VMMDev is not presently
5521	* mapped (e.g. EFI32).
5522	*
5523	* @param pVCpu The cross context virtual CPU structure.
5524	* @param pVmxTransient The VMX-transient structure.
5525	*
5526	* @remarks No-long-jump zone!!!
5527	*/
5528	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5529	{
5530	int rc = VINF_SUCCESS;
5531	PVM pVM = pVCpu->CTX_SUFF(pVM);
5532
5533	/*
5534	* Guest CR2.
5535	* It's always loaded in the assembler code. Nothing to do here.
5536	*/
5537
5538	/*
5539	* Guest CR3.
5540	*/
5541	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5542	{
5543	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5544
5545	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5546	if (pVM->hm.s.fNestedPaging)
5547	{
5548	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5549	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5550
5551	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5552	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5553	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5554	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5555
5556	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5557	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5558	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5559
5560	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5561	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5562	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5563	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5564	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5565	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5566	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5567
5568	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5569	AssertRCReturn(rc, rc);
5570
5571	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5572	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5573	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5574	{
5575	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5576	if (CPUMIsGuestInPAEModeEx(pCtx))
5577	{
5578	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5579	AssertRCReturn(rc, rc);
5580	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5581	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5582	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5583	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5584	AssertRCReturn(rc, rc);
5585	}
5586
5587	/*
5588	* The guest's view of its CR3 is unblemished with nested paging when the
5589	* guest is using paging or we have unrestricted guest execution to handle
5590	* the guest when it's not using paging.
5591	*/
5592	GCPhysGuestCr3 = pCtx->cr3;
5593	}
5594	else
5595	{
5596	/*
5597	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5598	* thinks it accesses physical memory directly, we use our identity-mapped
5599	* page table to map guest-linear to guest-physical addresses. EPT takes care
5600	* of translating it to host-physical addresses.
5601	*/
5602	RTGCPHYS GCPhys;
5603	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5604
5605	/* We obtain it here every time as the guest could have relocated this PCI region. */
5606	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5607	if (RT_SUCCESS(rc))
5608	{ /* likely */ }
5609	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5610	{
5611	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5612	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5613	}
5614	else
5615	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5616
5617	GCPhysGuestCr3 = GCPhys;
5618	}
5619
5620	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5621	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5622	AssertRCReturn(rc, rc);
5623	}
5624	else
5625	{
5626	/* Non-nested paging case, just use the hypervisor's CR3. */
5627	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5628
5629	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5630	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5631	AssertRCReturn(rc, rc);
5632	}
5633
5634	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5635	}
5636
5637	/*
5638	* Guest CR4.
5639	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5640	*/
5641	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5642	{
5643	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5644	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5645
5646	/*
5647	* Figure out fixed CR4 bits in VMX operation.
5648	*/
5649	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5650	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5651	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5652
5653	/*
5654	* With nested-guests, we may have extended the guest/host mask here (since we
5655	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5656	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5657	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5658	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5659	*/
5660	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5661	uint64_t u64GuestCr4 = pCtx->cr4;
5662	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5663	? pCtx->cr4
5664	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5665	Assert(!RT_HI_U32(u64GuestCr4));
5666
5667	/*
5668	* Setup VT-x's view of the guest CR4.
5669	*
5670	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5671	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5672	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5673	*
5674	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5675	*/
5676	if (pVmcsInfo->RealMode.fRealOnV86Active)
5677	{
5678	Assert(pVM->hm.s.vmx.pRealModeTSS);
5679	Assert(PDMVmmDevHeapIsEnabled(pVM));
5680	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5681	}
5682
5683	if (pVM->hm.s.fNestedPaging)
5684	{
5685	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5686	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5687	{
5688	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5689	u64GuestCr4 \|= X86_CR4_PSE;
5690	/* Our identity mapping is a 32-bit page directory. */
5691	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5692	}
5693	/* else use guest CR4.*/
5694	}
5695	else
5696	{
5697	Assert(!pVmxTransient->fIsNestedGuest);
5698
5699	/*
5700	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5701	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5702	*/
5703	switch (pVCpu->hm.s.enmShadowMode)
5704	{
5705	case PGMMODE_REAL: /* Real-mode. */
5706	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5707	case PGMMODE_32_BIT: /* 32-bit paging. */
5708	{
5709	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5710	break;
5711	}
5712
5713	case PGMMODE_PAE: /* PAE paging. */
5714	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5715	{
5716	u64GuestCr4 \|= X86_CR4_PAE;
5717	break;
5718	}
5719
5720	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5721	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5722	#ifdef VBOX_WITH_64_BITS_GUESTS
5723	break;
5724	#endif
5725	default:
5726	AssertFailed();
5727	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5728	}
5729	}
5730
5731	/*
5732	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
5733	* Note! For nested-guests, we could be altering our VMX emulation's
5734	* fixed bits. We thus need to re-apply them while importing CR4.
5735	*/
5736	u64GuestCr4 \|= fSetCr4;
5737	u64GuestCr4 &= fZapCr4;
5738
5739	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5740	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5741	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5742	AssertRCReturn(rc, rc);
5743
5744	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5745	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5746
5747	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5748
5749	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5750	}
5751	return rc;
5752	}
5753
5754
5755	/**
5756	* Exports the guest debug registers into the guest-state area in the VMCS.
5757	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5758	*
5759	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5760	*
5761	* @returns VBox status code.
5762	* @param pVCpu The cross context virtual CPU structure.
5763	* @param pVmxTransient The VMX-transient structure.
5764	*
5765	* @remarks No-long-jump zone!!!
5766	*/
5767	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5768	{
5769	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5770
5771	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5772	* stepping. */
5773	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5774	if (pVmxTransient->fIsNestedGuest)
5775	{
5776	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5777	AssertRCReturn(rc, rc);
5778
5779	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5780	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5781	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5782	AssertRCReturn(rc, rc);
5783	return VINF_SUCCESS;
5784	}
5785
5786	#ifdef VBOX_STRICT
5787	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5788	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5789	{
5790	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5791	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5792	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5793	}
5794	#endif
5795
5796	bool fSteppingDB = false;
5797	bool fInterceptMovDRx = false;
5798	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5799	if (pVCpu->hm.s.fSingleInstruction)
5800	{
5801	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5802	PVM pVM = pVCpu->CTX_SUFF(pVM);
5803	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5804	{
5805	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5806	Assert(fSteppingDB == false);
5807	}
5808	else
5809	{
5810	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5811	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5812	pVCpu->hm.s.fClearTrapFlag = true;
5813	fSteppingDB = true;
5814	}
5815	}
5816
5817	uint64_t u64GuestDr7;
5818	if ( fSteppingDB
5819	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5820	{
5821	/*
5822	* Use the combined guest and host DRx values found in the hypervisor register set
5823	* because the hypervisor debugger has breakpoints active or someone is single stepping
5824	* on the host side without a monitor trap flag.
5825	*
5826	* Note! DBGF expects a clean DR6 state before executing guest code.
5827	*/
5828	if (!CPUMIsHyperDebugStateActive(pVCpu))
5829	{
5830	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5831	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5832	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5833	}
5834
5835	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5836	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
5837	pVCpu->hm.s.fUsingHyperDR7 = true;
5838	fInterceptMovDRx = true;
5839	}
5840	else
5841	{
5842	/*
5843	* If the guest has enabled debug registers, we need to load them prior to
5844	* executing guest code so they'll trigger at the right time.
5845	*/
5846	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5847	{
5848	if (!CPUMIsGuestDebugStateActive(pVCpu))
5849	{
5850	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5851	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5852	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5853	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5854	}
5855	Assert(!fInterceptMovDRx);
5856	}
5857	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5858	{
5859	/*
5860	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5861	* must intercept #DB in order to maintain a correct DR6 guest value, and
5862	* because we need to intercept it to prevent nested #DBs from hanging the
5863	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5864	*/
5865	fInterceptMovDRx = true;
5866	}
5867
5868	/* Update DR7 with the actual guest value. */
5869	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5870	pVCpu->hm.s.fUsingHyperDR7 = false;
5871	}
5872
5873	if (fInterceptMovDRx)
5874	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5875	else
5876	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5877
5878	/*
5879	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5880	* monitor-trap flag and update our cache.
5881	*/
5882	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5883	{
5884	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5885	AssertRCReturn(rc2, rc2);
5886	pVmcsInfo->u32ProcCtls = uProcCtls;
5887	}
5888
5889	/*
5890	* Update guest DR7.
5891	*/
5892	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
5893	AssertRCReturn(rc, rc);
5894
5895	/*
5896	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5897	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5898	*
5899	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5900	*/
5901	if (fSteppingDB)
5902	{
5903	Assert(pVCpu->hm.s.fSingleInstruction);
5904	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5905
5906	uint32_t fIntrState = 0;
5907	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5908	AssertRCReturn(rc, rc);
5909
5910	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5911	{
5912	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5913	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5914	AssertRCReturn(rc, rc);
5915	}
5916	}
5917
5918	return VINF_SUCCESS;
5919	}
5920
5921
5922	#ifdef VBOX_STRICT
5923	/**
5924	* Strict function to validate segment registers.
5925	*
5926	* @param pVCpu The cross context virtual CPU structure.
5927	* @param pVmcsInfo The VMCS info. object.
5928	*
5929	* @remarks Will import guest CR0 on strict builds during validation of
5930	* segments.
5931	*/
5932	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5933	{
5934	/*
5935	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5936	*
5937	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5938	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5939	* unusable bit and doesn't change the guest-context value.
5940	*/
5941	PVM pVM = pVCpu->CTX_SUFF(pVM);
5942	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5943	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5944	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5945	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5946	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5947	{
5948	/* Protected mode checks */
5949	/* CS */
5950	Assert(pCtx->cs.Attr.n.u1Present);
5951	Assert(!(pCtx->cs.Attr.u & 0xf00));
5952	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5953	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5954	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5955	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5956	\|\| (pCtx->cs.Attr.n.u1Granularity));
5957	/* CS cannot be loaded with NULL in protected mode. */
5958	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5959	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5960	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5961	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5962	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5963	else
5964	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5965	/* SS */
5966	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5967	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5968	if ( !(pCtx->cr0 & X86_CR0_PE)
5969	\|\| pCtx->cs.Attr.n.u4Type == 3)
5970	{
5971	Assert(!pCtx->ss.Attr.n.u2Dpl);
5972	}
5973	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5974	{
5975	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5976	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5977	Assert(pCtx->ss.Attr.n.u1Present);
5978	Assert(!(pCtx->ss.Attr.u & 0xf00));
5979	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5980	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5981	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5982	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5983	\|\| (pCtx->ss.Attr.n.u1Granularity));
5984	}
5985	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5986	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5987	{
5988	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5989	Assert(pCtx->ds.Attr.n.u1Present);
5990	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5991	Assert(!(pCtx->ds.Attr.u & 0xf00));
5992	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5993	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5994	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5995	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5996	\|\| (pCtx->ds.Attr.n.u1Granularity));
5997	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5998	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5999	}
6000	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6001	{
6002	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6003	Assert(pCtx->es.Attr.n.u1Present);
6004	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6005	Assert(!(pCtx->es.Attr.u & 0xf00));
6006	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6007	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6008	\|\| !(pCtx->es.Attr.n.u1Granularity));
6009	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6010	\|\| (pCtx->es.Attr.n.u1Granularity));
6011	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6012	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6013	}
6014	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6015	{
6016	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6017	Assert(pCtx->fs.Attr.n.u1Present);
6018	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6019	Assert(!(pCtx->fs.Attr.u & 0xf00));
6020	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6021	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6022	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6023	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6024	\|\| (pCtx->fs.Attr.n.u1Granularity));
6025	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6026	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6027	}
6028	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6029	{
6030	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6031	Assert(pCtx->gs.Attr.n.u1Present);
6032	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6033	Assert(!(pCtx->gs.Attr.u & 0xf00));
6034	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6035	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6036	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6037	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6038	\|\| (pCtx->gs.Attr.n.u1Granularity));
6039	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6040	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6041	}
6042	/* 64-bit capable CPUs. */
6043	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6044	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6045	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6046	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6047	}
6048	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6049	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6050	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6051	{
6052	/* Real and v86 mode checks. */
6053	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6054	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6055	if (pVmcsInfo->RealMode.fRealOnV86Active)
6056	{
6057	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6058	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6059	}
6060	else
6061	{
6062	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6063	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6064	}
6065
6066	/* CS */
6067	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6068	Assert(pCtx->cs.u32Limit == 0xffff);
6069	Assert(u32CSAttr == 0xf3);
6070	/* SS */
6071	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6072	Assert(pCtx->ss.u32Limit == 0xffff);
6073	Assert(u32SSAttr == 0xf3);
6074	/* DS */
6075	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6076	Assert(pCtx->ds.u32Limit == 0xffff);
6077	Assert(u32DSAttr == 0xf3);
6078	/* ES */
6079	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6080	Assert(pCtx->es.u32Limit == 0xffff);
6081	Assert(u32ESAttr == 0xf3);
6082	/* FS */
6083	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6084	Assert(pCtx->fs.u32Limit == 0xffff);
6085	Assert(u32FSAttr == 0xf3);
6086	/* GS */
6087	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6088	Assert(pCtx->gs.u32Limit == 0xffff);
6089	Assert(u32GSAttr == 0xf3);
6090	/* 64-bit capable CPUs. */
6091	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6092	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6093	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6094	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6095	}
6096	}
6097	#endif /* VBOX_STRICT */
6098
6099
6100	/**
6101	* Exports a guest segment register into the guest-state area in the VMCS.
6102	*
6103	* @returns VBox status code.
6104	* @param pVCpu The cross context virtual CPU structure.
6105	* @param pVmcsInfo The VMCS info. object.
6106	* @param iSegReg The segment register number (X86_SREG_XXX).
6107	* @param pSelReg Pointer to the segment selector.
6108	*
6109	* @remarks No-long-jump zone!!!
6110	*/
6111	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6112	{
6113	Assert(iSegReg < X86_SREG_COUNT);
6114	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6115	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6116	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6117	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6118
6119	uint32_t u32Access = pSelReg->Attr.u;
6120	if (pVmcsInfo->RealMode.fRealOnV86Active)
6121	{
6122	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6123	u32Access = 0xf3;
6124	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6125	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6126	RT_NOREF_PV(pVCpu);
6127	}
6128	else
6129	{
6130	/*
6131	* The way to differentiate between whether this is really a null selector or was just
6132	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6133	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6134	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6135	* NULL selectors loaded in protected-mode have their attribute as 0.
6136	*/
6137	if (!u32Access)
6138	u32Access = X86DESCATTR_UNUSABLE;
6139	}
6140
6141	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6142	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6143	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6144
6145	/*
6146	* Commit it to the VMCS.
6147	*/
6148	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6149	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6150	rc \|= VMXWriteVmcsNw(idxBase, pSelReg->u64Base);
6151	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6152	AssertRCReturn(rc, rc);
6153	return rc;
6154	}
6155
6156
6157	/**
6158	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6159	* area in the VMCS.
6160	*
6161	* @returns VBox status code.
6162	* @param pVCpu The cross context virtual CPU structure.
6163	* @param pVmxTransient The VMX-transient structure.
6164	*
6165	* @remarks Will import guest CR0 on strict builds during validation of
6166	* segments.
6167	* @remarks No-long-jump zone!!!
6168	*/
6169	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6170	{
6171	int rc = VERR_INTERNAL_ERROR_5;
6172	PVM pVM = pVCpu->CTX_SUFF(pVM);
6173	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6174	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6175
6176	/*
6177	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6178	*/
6179	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6180	{
6181	#ifdef VBOX_WITH_REM
6182	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6183	{
6184	Assert(!pVmxTransient->fIsNestedGuest);
6185	Assert(pVM->hm.s.vmx.pRealModeTSS);
6186	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6187	if ( pVmcsInfo->fWasInRealMode
6188	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6189	{
6190	/*
6191	* Notify the recompiler must flush its code-cache as the guest -may-
6192	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6193	*/
6194	REMFlushTBs(pVM);
6195	Log4Func(("Switch to protected mode detected!\n"));
6196	pVmcsInfo->fWasInRealMode = false;
6197	}
6198	}
6199	#endif
6200	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6201	{
6202	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6203	if (pVmcsInfo->RealMode.fRealOnV86Active)
6204	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6205	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6206	AssertRCReturn(rc, rc);
6207	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6208	}
6209
6210	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6211	{
6212	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6213	if (pVmcsInfo->RealMode.fRealOnV86Active)
6214	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6215	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6216	AssertRCReturn(rc, rc);
6217	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6218	}
6219
6220	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6221	{
6222	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6223	if (pVmcsInfo->RealMode.fRealOnV86Active)
6224	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6225	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6226	AssertRCReturn(rc, rc);
6227	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6228	}
6229
6230	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6231	{
6232	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6233	if (pVmcsInfo->RealMode.fRealOnV86Active)
6234	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6235	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6236	AssertRCReturn(rc, rc);
6237	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6238	}
6239
6240	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6241	{
6242	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6243	if (pVmcsInfo->RealMode.fRealOnV86Active)
6244	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6245	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6246	AssertRCReturn(rc, rc);
6247	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6248	}
6249
6250	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6251	{
6252	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6253	if (pVmcsInfo->RealMode.fRealOnV86Active)
6254	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6255	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6256	AssertRCReturn(rc, rc);
6257	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6258	}
6259
6260	#ifdef VBOX_STRICT
6261	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6262	#endif
6263	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6264	pCtx->cs.Attr.u));
6265	}
6266
6267	/*
6268	* Guest TR.
6269	*/
6270	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6271	{
6272	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6273
6274	/*
6275	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6276	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6277	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6278	*/
6279	uint16_t u16Sel;
6280	uint32_t u32Limit;
6281	uint64_t u64Base;
6282	uint32_t u32AccessRights;
6283	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6284	{
6285	u16Sel = pCtx->tr.Sel;
6286	u32Limit = pCtx->tr.u32Limit;
6287	u64Base = pCtx->tr.u64Base;
6288	u32AccessRights = pCtx->tr.Attr.u;
6289	}
6290	else
6291	{
6292	Assert(!pVmxTransient->fIsNestedGuest);
6293	Assert(pVM->hm.s.vmx.pRealModeTSS);
6294	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6295
6296	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6297	RTGCPHYS GCPhys;
6298	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6299	AssertRCReturn(rc, rc);
6300
6301	X86DESCATTR DescAttr;
6302	DescAttr.u = 0;
6303	DescAttr.n.u1Present = 1;
6304	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6305
6306	u16Sel = 0;
6307	u32Limit = HM_VTX_TSS_SIZE;
6308	u64Base = GCPhys;
6309	u32AccessRights = DescAttr.u;
6310	}
6311
6312	/* Validate. */
6313	Assert(!(u16Sel & RT_BIT(2)));
6314	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6315	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6316	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6317	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6318	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6319	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6320	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6321	Assert( (u32Limit & 0xfff) == 0xfff
6322	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6323	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6324	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6325
6326	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6327	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6328	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6329	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_TR_BASE, u64Base);
6330	AssertRCReturn(rc, rc);
6331
6332	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6333	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6334	}
6335
6336	/*
6337	* Guest GDTR.
6338	*/
6339	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6340	{
6341	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6342
6343	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6344	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6345	AssertRCReturn(rc, rc);
6346
6347	/* Validate. */
6348	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6349
6350	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6351	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6352	}
6353
6354	/*
6355	* Guest LDTR.
6356	*/
6357	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6358	{
6359	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6360
6361	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6362	uint32_t u32Access;
6363	if ( !pVmxTransient->fIsNestedGuest
6364	&& !pCtx->ldtr.Attr.u)
6365	u32Access = X86DESCATTR_UNUSABLE;
6366	else
6367	u32Access = pCtx->ldtr.Attr.u;
6368
6369	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6370	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6371	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6372	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6373	AssertRCReturn(rc, rc);
6374
6375	/* Validate. */
6376	if (!(u32Access & X86DESCATTR_UNUSABLE))
6377	{
6378	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6379	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6380	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6381	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6382	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6383	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6384	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6385	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6386	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6387	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6388	}
6389
6390	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6391	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6392	}
6393
6394	/*
6395	* Guest IDTR.
6396	*/
6397	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6398	{
6399	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6400
6401	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6402	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6403	AssertRCReturn(rc, rc);
6404
6405	/* Validate. */
6406	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6407
6408	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6409	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6410	}
6411
6412	return VINF_SUCCESS;
6413	}
6414
6415
6416	/**
6417	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6418	* areas.
6419	*
6420	* These MSRs will automatically be loaded to the host CPU on every successful
6421	* VM-entry and stored from the host CPU on every successful VM-exit.
6422	*
6423	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6424	* actual host MSR values are not- updated here for performance reasons. See
6425	* hmR0VmxExportHostMsrs().
6426	*
6427	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6428	*
6429	* @returns VBox status code.
6430	* @param pVCpu The cross context virtual CPU structure.
6431	* @param pVmxTransient The VMX-transient structure.
6432	*
6433	* @remarks No-long-jump zone!!!
6434	*/
6435	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6436	{
6437	AssertPtr(pVCpu);
6438	AssertPtr(pVmxTransient);
6439
6440	PVM pVM = pVCpu->CTX_SUFF(pVM);
6441	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6442
6443	/*
6444	* MSRs that we use the auto-load/store MSR area in the VMCS.
6445	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
6446	* nothing to do here. The host MSR values are updated when it's safe in
6447	* hmR0VmxLazySaveHostMsrs().
6448	*
6449	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6450	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6451	* emulation, nothing to do here.
6452	*/
6453	/** @todo sort out HM_CHANGED_VMX_GUEST_AUTO_MSRS. */
6454	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6455	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6456
6457	/*
6458	* Guest Sysenter MSRs.
6459	*/
6460	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6461	{
6462	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6463
6464	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6465	{
6466	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6467	AssertRCReturn(rc, rc);
6468	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6469	}
6470
6471	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6472	{
6473	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6474	AssertRCReturn(rc, rc);
6475	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6476	}
6477
6478	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6479	{
6480	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6481	AssertRCReturn(rc, rc);
6482	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6483	}
6484	}
6485
6486	/*
6487	* Guest/host EFER MSR.
6488	*/
6489	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6490	{
6491	/* Whether we are using the VMCS to swap the EFER MSR must have been
6492	determined earlier while exporting VM-entry/VM-exit controls. */
6493	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6494	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6495
6496	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6497	{
6498	/*
6499	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6500	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6501	*/
6502	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6503	{
6504	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6505	AssertRCReturn(rc, rc);
6506	}
6507	else
6508	{
6509	/*
6510	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6511	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6512	*/
6513	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6514	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6515	AssertRCReturn(rc, rc);
6516	}
6517	}
6518	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6519	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6520
6521	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6522	}
6523
6524	/*
6525	* Other MSRs.
6526	* Speculation Control (R/W).
6527	*/
6528	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6529	{
6530	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6531	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6532	{
6533	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6534	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6535	AssertRCReturn(rc, rc);
6536	}
6537	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6538	}
6539
6540	return VINF_SUCCESS;
6541	}
6542
6543
6544	/**
6545	* Selects up the appropriate function to run guest code.
6546	*
6547	* @returns VBox status code.
6548	* @param pVCpu The cross context virtual CPU structure.
6549	* @param pVmxTransient The VMX-transient structure.
6550	*
6551	* @remarks No-long-jump zone!!!
6552	*/
6553	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6554	{
6555	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6556	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6557
6558	if (CPUMIsGuestInLongModeEx(pCtx))
6559	{
6560	#ifndef VBOX_WITH_64_BITS_GUESTS
6561	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6562	#else
6563	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6564	/* Guest is in long mode, use the 64-bit handler (host is 64-bit). */
6565	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6566	#endif
6567	}
6568	else
6569	{
6570	/* Guest is not in long mode, use the 32-bit handler. */
6571	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6572	}
6573	Assert(pVmcsInfo->pfnStartVM);
6574	return VINF_SUCCESS;
6575	}
6576
6577
6578	/**
6579	* Wrapper for running the guest code in VT-x.
6580	*
6581	* @returns VBox status code, no informational status codes.
6582	* @param pVCpu The cross context virtual CPU structure.
6583	* @param pVmxTransient The VMX-transient structure.
6584	*
6585	* @remarks No-long-jump zone!!!
6586	*/
6587	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6588	{
6589	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6590	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6591	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6592
6593	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6594
6595	/*
6596	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6597	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6598	* callee-saved and thus the need for this XMM wrapper.
6599	*
6600	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6601	*/
6602	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6603	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6604	PVM pVM = pVCpu->CTX_SUFF(pVM);
6605	#ifdef VBOX_WITH_KERNEL_USING_XMM
6606	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6607	#else
6608	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6609	#endif
6610	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6611	return rc;
6612	}
6613
6614
6615	/**
6616	* Reports world-switch error and dumps some useful debug info.
6617	*
6618	* @param pVCpu The cross context virtual CPU structure.
6619	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6620	* @param pVmxTransient The VMX-transient structure (only
6621	* exitReason updated).
6622	*/
6623	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6624	{
6625	Assert(pVCpu);
6626	Assert(pVmxTransient);
6627	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6628
6629	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6630	switch (rcVMRun)
6631	{
6632	case VERR_VMX_INVALID_VMXON_PTR:
6633	AssertFailed();
6634	break;
6635	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6636	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6637	{
6638	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6639	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6640	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
6641	AssertRC(rc);
6642
6643	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6644	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6645	Cannot do it here as we may have been long preempted. */
6646
6647	#ifdef VBOX_STRICT
6648	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6649	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6650	pVmxTransient->uExitReason));
6651	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6652	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6653	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6654	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6655	else
6656	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6657	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6658	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6659
6660	static struct
6661	{
6662	/** Name of the field to log. */
6663	const char *pszName;
6664	/** The VMCS field. */
6665	uint32_t uVmcsField;
6666	/** Whether host support of this field needs to be checked. */
6667	bool fCheckSupport;
6668	} const s_aVmcsFields[] =
6669	{
6670	{ "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false, },
6671	{ "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false, },
6672	{ "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true, },
6673	{ "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false, },
6674	{ "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false, },
6675	{ "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false, },
6676	{ "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false, },
6677	{ "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false, },
6678	{ "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false, },
6679	{ "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false, },
6680	{ "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false, },
6681	{ "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false, },
6682	{ "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false, },
6683	{ "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false, },
6684	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false, },
6685	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false, },
6686	{ "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false, },
6687	{ "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false, },
6688	{ "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false, },
6689	{ "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false, },
6690	{ "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true, },
6691	{ "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false, },
6692	{ "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false, },
6693	{ "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false, },
6694	{ "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
6695	{ "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false, },
6696	{ "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false, },
6697	{ "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false, },
6698	/* The order of selector fields below are fixed! */
6699	{ "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false, },
6700	{ "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false, },
6701	{ "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false, },
6702	{ "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false, },
6703	{ "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false, },
6704	{ "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false, },
6705	{ "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false, },
6706	/* End of ordered selector fields. */
6707	{ "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false, },
6708	{ "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false, },
6709	{ "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false, },
6710	{ "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false, },
6711	{ "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false, },
6712	{ "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false, },
6713	{ "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false, },
6714	{ "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false, }
6715	};
6716
6717	RTGDTR HostGdtr;
6718	ASMGetGDTR(&HostGdtr);
6719
6720	uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
6721	for (uint32_t i = 0; i < cVmcsFields; i++)
6722	{
6723	uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
6724
6725	bool fSupported;
6726	if (!s_aVmcsFields[i].fCheckSupport)
6727	fSupported = true;
6728	else
6729	{
6730	PVM pVM = pVCpu->CTX_SUFF(pVM);
6731	switch (uVmcsField)
6732	{
6733	case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hm.s.fNestedPaging; break;
6734	case VMX_VMCS16_VPID: fSupported = pVM->hm.s.vmx.fVpid; break;
6735	case VMX_VMCS32_CTRL_PROC_EXEC2:
6736	fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6737	break;
6738	default:
6739	AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
6740	}
6741	}
6742
6743	if (fSupported)
6744	{
6745	uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
6746	switch (uWidth)
6747	{
6748	case VMX_VMCSFIELD_WIDTH_16BIT:
6749	{
6750	uint16_t u16Val;
6751	rc = VMXReadVmcs16(uVmcsField, &u16Val);
6752	AssertRC(rc);
6753	Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
6754
6755	if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
6756	&& uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
6757	{
6758	if (u16Val < HostGdtr.cbGdt)
6759	{
6760	/* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
6761	static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
6762	"Host FS", "Host GS", "Host TR" };
6763	uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
6764	Assert(idxSel < RT_ELEMENTS(s_apszSel));
6765	PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
6766	hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
6767	}
6768	else
6769	Log4((" Selector value exceeds GDT limit!\n"));
6770	}
6771	break;
6772	}
6773
6774	case VMX_VMCSFIELD_WIDTH_32BIT:
6775	{
6776	uint32_t u32Val;
6777	rc = VMXReadVmcs32(uVmcsField, &u32Val);
6778	AssertRC(rc);
6779	Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
6780	break;
6781	}
6782
6783	case VMX_VMCSFIELD_WIDTH_64BIT:
6784	case VMX_VMCSFIELD_WIDTH_NATURAL:
6785	{
6786	uint64_t u64Val;
6787	rc = VMXReadVmcs64(uVmcsField, &u64Val);
6788	AssertRC(rc);
6789	Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
6790	break;
6791	}
6792	}
6793	}
6794	}
6795
6796	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6797	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6798	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6799	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6800	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6801	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6802	#endif /* VBOX_STRICT */
6803	break;
6804	}
6805
6806	default:
6807	/* Impossible */
6808	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6809	break;
6810	}
6811	}
6812
6813
6814	/**
6815	* Sets up the usage of TSC-offsetting and updates the VMCS.
6816	*
6817	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6818	* VMX-preemption timer.
6819	*
6820	* @returns VBox status code.
6821	* @param pVCpu The cross context virtual CPU structure.
6822	* @param pVmxTransient The VMX-transient structure.
6823	*
6824	* @remarks No-long-jump zone!!!
6825	*/
6826	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6827	{
6828	bool fOffsettedTsc;
6829	bool fParavirtTsc;
6830	uint64_t uTscOffset;
6831	PVM pVM = pVCpu->CTX_SUFF(pVM);
6832	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6833
6834	if (pVM->hm.s.vmx.fUsePreemptTimer)
6835	{
6836	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6837
6838	/* Make sure the returned values have sane upper and lower boundaries. */
6839	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6840	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6841	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6842	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6843
6844	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6845	* preemption timers here. We probably need to clamp the preemption timer,
6846	* after converting the timer value to the host. */
6847	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6848	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6849	AssertRC(rc);
6850	}
6851	else
6852	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6853
6854	if (fParavirtTsc)
6855	{
6856	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6857	information before every VM-entry, hence disable it for performance sake. */
6858	#if 0
6859	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6860	AssertRC(rc);
6861	#endif
6862	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6863	}
6864
6865	if ( fOffsettedTsc
6866	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6867	{
6868	if (pVmxTransient->fIsNestedGuest)
6869	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6870	hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
6871	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6872	}
6873	else
6874	{
6875	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6876	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6877	}
6878	}
6879
6880
6881	/**
6882	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6883	* VM-exit interruption info type.
6884	*
6885	* @returns The IEM exception flags.
6886	* @param uVector The event vector.
6887	* @param uVmxEventType The VMX event type.
6888	*
6889	* @remarks This function currently only constructs flags required for
6890	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6891	* and CR2 aspects of an exception are not included).
6892	*/
6893	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6894	{
6895	uint32_t fIemXcptFlags;
6896	switch (uVmxEventType)
6897	{
6898	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6899	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6900	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6901	break;
6902
6903	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6904	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6905	break;
6906
6907	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6908	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6909	break;
6910
6911	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6912	{
6913	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6914	if (uVector == X86_XCPT_BP)
6915	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6916	else if (uVector == X86_XCPT_OF)
6917	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6918	else
6919	{
6920	fIemXcptFlags = 0;
6921	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6922	}
6923	break;
6924	}
6925
6926	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6927	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6928	break;
6929
6930	default:
6931	fIemXcptFlags = 0;
6932	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6933	break;
6934	}
6935	return fIemXcptFlags;
6936	}
6937
6938
6939	/**
6940	* Sets an event as a pending event to be injected into the guest.
6941	*
6942	* @param pVCpu The cross context virtual CPU structure.
6943	* @param u32IntInfo The VM-entry interruption-information field.
6944	* @param cbInstr The VM-entry instruction length in bytes (for software
6945	* interrupts, exceptions and privileged software
6946	* exceptions).
6947	* @param u32ErrCode The VM-entry exception error code.
6948	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6949	* page-fault.
6950	*/
6951	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6952	RTGCUINTPTR GCPtrFaultAddress)
6953	{
6954	Assert(!pVCpu->hm.s.Event.fPending);
6955	pVCpu->hm.s.Event.fPending = true;
6956	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6957	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6958	pVCpu->hm.s.Event.cbInstr = cbInstr;
6959	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6960	}
6961
6962
6963	/**
6964	* Sets an external interrupt as pending-for-injection into the VM.
6965	*
6966	* @param pVCpu The cross context virtual CPU structure.
6967	* @param u8Interrupt The external interrupt vector.
6968	*/
6969	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6970	{
6971	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6972	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6973	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6974	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6975	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6976	}
6977
6978
6979	/**
6980	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6981	*
6982	* @param pVCpu The cross context virtual CPU structure.
6983	*/
6984	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6985	{
6986	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6987	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6988	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6989	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6990	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6991	}
6992
6993
6994	/**
6995	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6996	*
6997	* @param pVCpu The cross context virtual CPU structure.
6998	*/
6999	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7000	{
7001	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7002	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7003	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7004	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7005	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7006	}
7007
7008
7009	/**
7010	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7011	*
7012	* @param pVCpu The cross context virtual CPU structure.
7013	*/
7014	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7015	{
7016	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7017	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7018	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7019	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7020	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7021	}
7022
7023
7024	/**
7025	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7026	*
7027	* @param pVCpu The cross context virtual CPU structure.
7028	*/
7029	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7030	{
7031	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7032	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7033	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7034	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7035	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7036	}
7037
7038
7039	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7040	/**
7041	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7042	*
7043	* @param pVCpu The cross context virtual CPU structure.
7044	* @param u32ErrCode The error code for the general-protection exception.
7045	*/
7046	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7047	{
7048	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7049	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7050	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7051	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7052	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7053	}
7054
7055
7056	/**
7057	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7058	*
7059	* @param pVCpu The cross context virtual CPU structure.
7060	* @param u32ErrCode The error code for the stack exception.
7061	*/
7062	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7063	{
7064	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7065	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7066	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7067	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7068	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7069	}
7070	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7071
7072
7073	/**
7074	* Fixes up attributes for the specified segment register.
7075	*
7076	* @param pVCpu The cross context virtual CPU structure.
7077	* @param pSelReg The segment register that needs fixing.
7078	* @param idxSel The VMCS field for the corresponding segment register.
7079	*/
7080	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7081	{
7082	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7083
7084	/*
7085	* If VT-x marks the segment as unusable, most other bits remain undefined:
7086	* - For CS the L, D and G bits have meaning.
7087	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7088	* - For the remaining data segments no bits are defined.
7089	*
7090	* The present bit and the unusable bit has been observed to be set at the
7091	* same time (the selector was supposed to be invalid as we started executing
7092	* a V8086 interrupt in ring-0).
7093	*
7094	* What should be important for the rest of the VBox code, is that the P bit is
7095	* cleared. Some of the other VBox code recognizes the unusable bit, but
7096	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7097	* safe side here, we'll strip off P and other bits we don't care about. If
7098	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7099	*
7100	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7101	*/
7102	#ifdef VBOX_STRICT
7103	uint32_t const uAttr = pSelReg->Attr.u;
7104	#endif
7105
7106	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7107	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7108	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7109
7110	#ifdef VBOX_STRICT
7111	VMMRZCallRing3Disable(pVCpu);
7112	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7113	# ifdef DEBUG_bird
7114	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7115	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7116	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7117	# endif
7118	VMMRZCallRing3Enable(pVCpu);
7119	NOREF(uAttr);
7120	#endif
7121	RT_NOREF2(pVCpu, idxSel);
7122	}
7123
7124
7125	/**
7126	* Imports a guest segment register from the current VMCS into the guest-CPU
7127	* context.
7128	*
7129	* @returns VBox status code.
7130	* @param pVCpu The cross context virtual CPU structure.
7131	* @param iSegReg The segment register number (X86_SREG_XXX).
7132	*
7133	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7134	* do not log!
7135	*/
7136	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7137	{
7138	Assert(iSegReg < X86_SREG_COUNT);
7139
7140	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7141	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7142	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7143	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7144
7145	uint16_t u16Sel;
7146	uint64_t u64Base;
7147	uint32_t u32Limit, u32Attr;
7148	int rc = VMXReadVmcs16(idxSel, &u16Sel);
7149	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7150	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7151	rc \|= VMXReadVmcsNw(idxBase, &u64Base);
7152	if (RT_SUCCESS(rc))
7153	{
7154	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7155	pSelReg->Sel = u16Sel;
7156	pSelReg->ValidSel = u16Sel;
7157	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7158	pSelReg->u32Limit = u32Limit;
7159	pSelReg->u64Base = u64Base;
7160	pSelReg->Attr.u = u32Attr;
7161	if (u32Attr & X86DESCATTR_UNUSABLE)
7162	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7163	}
7164	return rc;
7165	}
7166
7167
7168	/**
7169	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7170	*
7171	* @returns VBox status code.
7172	* @param pVCpu The cross context virtual CPU structure.
7173	*
7174	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7175	* do not log!
7176	*/
7177	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7178	{
7179	uint16_t u16Sel;
7180	uint64_t u64Base;
7181	uint32_t u32Limit, u32Attr;
7182	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel);
7183	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7184	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7185	rc \|= VMXReadVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7186	if (RT_SUCCESS(rc))
7187	{
7188	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
7189	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
7190	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7191	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7192	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7193	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7194	if (u32Attr & X86DESCATTR_UNUSABLE)
7195	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7196	}
7197	return rc;
7198	}
7199
7200
7201	/**
7202	* Imports the guest TR from the current VMCS into the guest-CPU context.
7203	*
7204	* @returns VBox status code.
7205	* @param pVCpu The cross context virtual CPU structure.
7206	*
7207	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7208	* do not log!
7209	*/
7210	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7211	{
7212	uint16_t u16Sel;
7213	uint64_t u64Base;
7214	uint32_t u32Limit, u32Attr;
7215	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL, &u16Sel);
7216	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7217	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7218	rc \|= VMXReadVmcsNw(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7219	AssertRCReturn(rc, rc);
7220
7221	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
7222	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
7223	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7224	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7225	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7226	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7227	/* TR is the only selector that can never be unusable. */
7228	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7229	return VINF_SUCCESS;
7230	}
7231
7232
7233	/**
7234	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7235	*
7236	* @returns VBox status code.
7237	* @param pVCpu The cross context virtual CPU structure.
7238	*
7239	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7240	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7241	* instead!!!
7242	*/
7243	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7244	{
7245	uint64_t u64Val;
7246	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7247	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7248	{
7249	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
7250	if (RT_SUCCESS(rc))
7251	{
7252	pCtx->rip = u64Val;
7253	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7254	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7255	}
7256	return rc;
7257	}
7258	return VINF_SUCCESS;
7259	}
7260
7261
7262	/**
7263	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7264	*
7265	* @returns VBox status code.
7266	* @param pVCpu The cross context virtual CPU structure.
7267	* @param pVmcsInfo The VMCS info. object.
7268	*
7269	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7270	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7271	* instead!!!
7272	*/
7273	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7274	{
7275	uint64_t u64Val;
7276	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7277	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7278	{
7279	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7280	if (RT_SUCCESS(rc))
7281	{
7282	pCtx->rflags.u64 = u64Val;
7283
7284	/* Restore eflags for real-on-v86-mode hack. */
7285	if (pVmcsInfo->RealMode.fRealOnV86Active)
7286	{
7287	pCtx->eflags.Bits.u1VM = 0;
7288	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7289	}
7290	}
7291	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7292	return rc;
7293	}
7294	return VINF_SUCCESS;
7295	}
7296
7297
7298	/**
7299	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7300	* context.
7301	*
7302	* @returns VBox status code.
7303	* @param pVCpu The cross context virtual CPU structure.
7304	* @param pVmcsInfo The VMCS info. object.
7305	*
7306	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7307	* do not log!
7308	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7309	* instead!!!
7310	*/
7311	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7312	{
7313	uint32_t u32Val;
7314	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7315	if (RT_SUCCESS(rc))
7316	{
7317	if (!u32Val)
7318	{
7319	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7320	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7321
7322	CPUMSetGuestNmiBlocking(pVCpu, false);
7323	}
7324	else
7325	{
7326	/*
7327	* We must import RIP here to set our EM interrupt-inhibited state.
7328	* We also import RFLAGS as our code that evaluates pending interrupts
7329	* before VM-entry requires it.
7330	*/
7331	rc = hmR0VmxImportGuestRip(pVCpu);
7332	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7333	if (RT_SUCCESS(rc))
7334	{
7335	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7336	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7337	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7338	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7339
7340	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7341	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7342	}
7343	}
7344	}
7345	return rc;
7346	}
7347
7348
7349	/**
7350	* Worker for VMXR0ImportStateOnDemand.
7351	*
7352	* @returns VBox status code.
7353	* @param pVCpu The cross context virtual CPU structure.
7354	* @param pVmcsInfo The VMCS info. object.
7355	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7356	*/
7357	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7358	{
7359	#define VMXLOCAL_BREAK_RC(a_rc) \
7360	if (RT_SUCCESS(a_rc)) \
7361	{ } \
7362	else \
7363	break
7364
7365	int rc = VINF_SUCCESS;
7366	PVM pVM = pVCpu->CTX_SUFF(pVM);
7367	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7368	uint64_t u64Val;
7369	uint32_t u32Val;
7370
7371	/*
7372	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7373	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7374	* neither are other host platforms.
7375	*
7376	* Committing this temporarily as it prevents BSOD.
7377	*
7378	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7379	*/
7380	#ifdef RT_OS_WINDOWS
7381	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7382	return VERR_HM_IPE_1;
7383	#endif
7384
7385	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7386
7387	/*
7388	* We disable interrupts to make the updating of the state and in particular
7389	* the fExtrn modification atomic wrt to preemption hooks.
7390	*/
7391	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7392
7393	fWhat &= pCtx->fExtrn;
7394	if (fWhat)
7395	{
7396	do
7397	{
7398	if (fWhat & CPUMCTX_EXTRN_RIP)
7399	{
7400	rc = hmR0VmxImportGuestRip(pVCpu);
7401	VMXLOCAL_BREAK_RC(rc);
7402	}
7403
7404	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7405	{
7406	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7407	VMXLOCAL_BREAK_RC(rc);
7408	}
7409
7410	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7411	{
7412	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7413	VMXLOCAL_BREAK_RC(rc);
7414	}
7415
7416	if (fWhat & CPUMCTX_EXTRN_RSP)
7417	{
7418	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &u64Val);
7419	VMXLOCAL_BREAK_RC(rc);
7420	pCtx->rsp = u64Val;
7421	}
7422
7423	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7424	{
7425	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7426	if (fWhat & CPUMCTX_EXTRN_CS)
7427	{
7428	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7429	rc \|= hmR0VmxImportGuestRip(pVCpu);
7430	if (fRealOnV86Active)
7431	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7432	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7433	}
7434	if (fWhat & CPUMCTX_EXTRN_SS)
7435	{
7436	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7437	if (fRealOnV86Active)
7438	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7439	}
7440	if (fWhat & CPUMCTX_EXTRN_DS)
7441	{
7442	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7443	if (fRealOnV86Active)
7444	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7445	}
7446	if (fWhat & CPUMCTX_EXTRN_ES)
7447	{
7448	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7449	if (fRealOnV86Active)
7450	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7451	}
7452	if (fWhat & CPUMCTX_EXTRN_FS)
7453	{
7454	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7455	if (fRealOnV86Active)
7456	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7457	}
7458	if (fWhat & CPUMCTX_EXTRN_GS)
7459	{
7460	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7461	if (fRealOnV86Active)
7462	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7463	}
7464	VMXLOCAL_BREAK_RC(rc);
7465	}
7466
7467	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7468	{
7469	if (fWhat & CPUMCTX_EXTRN_LDTR)
7470	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7471
7472	if (fWhat & CPUMCTX_EXTRN_GDTR)
7473	{
7474	rc \|= VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7475	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7476	pCtx->gdtr.pGdt = u64Val;
7477	pCtx->gdtr.cbGdt = u32Val;
7478	}
7479
7480	/* Guest IDTR. */
7481	if (fWhat & CPUMCTX_EXTRN_IDTR)
7482	{
7483	rc \|= VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7484	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7485	pCtx->idtr.pIdt = u64Val;
7486	pCtx->idtr.cbIdt = u32Val;
7487	}
7488
7489	/* Guest TR. */
7490	if (fWhat & CPUMCTX_EXTRN_TR)
7491	{
7492	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7493	don't need to import that one. */
7494	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7495	rc \|= hmR0VmxImportGuestTr(pVCpu);
7496	}
7497	VMXLOCAL_BREAK_RC(rc);
7498	}
7499
7500	if (fWhat & CPUMCTX_EXTRN_DR7)
7501	{
7502	if (!pVCpu->hm.s.fUsingHyperDR7)
7503	{
7504	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7505	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &u64Val);
7506	VMXLOCAL_BREAK_RC(rc);
7507	pCtx->dr[7] = u64Val;
7508	}
7509	}
7510
7511	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7512	{
7513	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7514	rc \|= VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7515	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7516	pCtx->SysEnter.cs = u32Val;
7517	VMXLOCAL_BREAK_RC(rc);
7518	}
7519
7520	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7521	{
7522	if ( pVM->hm.s.fAllow64BitGuests
7523	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7524	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7525	}
7526
7527	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7528	{
7529	if ( pVM->hm.s.fAllow64BitGuests
7530	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7531	{
7532	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7533	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7534	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7535	}
7536	}
7537
7538	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7539	{
7540	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7541	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7542	Assert(pMsrs);
7543	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7544	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7545	for (uint32_t i = 0; i < cMsrs; i++)
7546	{
7547	uint32_t const idMsr = pMsrs[i].u32Msr;
7548	switch (idMsr)
7549	{
7550	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7551	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7552	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7553	default:
7554	{
7555	pCtx->fExtrn = 0;
7556	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7557	ASMSetFlags(fEFlags);
7558	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7559	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7560	}
7561	}
7562	}
7563	}
7564
7565	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7566	{
7567	if (fWhat & CPUMCTX_EXTRN_CR0)
7568	{
7569	uint64_t u64Shadow;
7570	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Val);
7571	rc \|= VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7572	VMXLOCAL_BREAK_RC(rc);
7573	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7574	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7575
7576	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7577	/*
7578	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
7579	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
7580	*/
7581	if (CPUMIsGuestInVmxNonRootMode(pCtx))
7582	{
7583	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
7584	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
7585	}
7586	#endif
7587
7588	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7589	CPUMSetGuestCR0(pVCpu, u64Val);
7590	VMMRZCallRing3Enable(pVCpu);
7591	}
7592
7593	if (fWhat & CPUMCTX_EXTRN_CR4)
7594	{
7595	uint64_t u64Shadow;
7596	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64Val);
7597	rc \|= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7598	VMXLOCAL_BREAK_RC(rc);
7599	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7600	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7601
7602	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7603	/*
7604	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
7605	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
7606	*/
7607	if (CPUMIsGuestInVmxNonRootMode(pCtx))
7608	{
7609	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
7610	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
7611	}
7612	#endif
7613
7614	pCtx->cr4 = u64Val;
7615	}
7616
7617	if (fWhat & CPUMCTX_EXTRN_CR3)
7618	{
7619	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7620	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7621	\|\| ( pVM->hm.s.fNestedPaging
7622	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7623	{
7624	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Val);
7625	VMXLOCAL_BREAK_RC(rc);
7626	if (pCtx->cr3 != u64Val)
7627	{
7628	pCtx->cr3 = u64Val;
7629	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7630	}
7631
7632	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7633	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7634	if (CPUMIsGuestInPAEModeEx(pCtx))
7635	{
7636	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7637	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7638	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7639	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7640	VMXLOCAL_BREAK_RC(rc);
7641	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7642	}
7643	}
7644	}
7645	}
7646
7647	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7648	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7649	{
7650	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7651	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7652	{
7653	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7654	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7655	VMXLOCAL_BREAK_RC(rc);
7656	}
7657	}
7658	#endif
7659	} while (0);
7660
7661	if (RT_SUCCESS(rc))
7662	{
7663	/* Update fExtrn. */
7664	pCtx->fExtrn &= ~fWhat;
7665
7666	/* If everything has been imported, clear the HM keeper bit. */
7667	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7668	{
7669	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7670	Assert(!pCtx->fExtrn);
7671	}
7672	}
7673	}
7674	else
7675	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7676
7677	ASMSetFlags(fEFlags);
7678
7679	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7680
7681	if (RT_SUCCESS(rc))
7682	{ /* likely */ }
7683	else
7684	return rc;
7685
7686	/*
7687	* Honor any pending CR3 updates.
7688	*
7689	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7690	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7691	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7692	*
7693	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7694	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7695	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7696	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7697	*
7698	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7699	*/
7700	if (VMMRZCallRing3IsEnabled(pVCpu))
7701	{
7702	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7703	{
7704	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7705	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7706	}
7707
7708	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7709	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7710
7711	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7712	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7713	}
7714
7715	return VINF_SUCCESS;
7716	#undef VMXLOCAL_BREAK_RC
7717	}
7718
7719
7720	/**
7721	* Saves the guest state from the VMCS into the guest-CPU context.
7722	*
7723	* @returns VBox status code.
7724	* @param pVCpu The cross context virtual CPU structure.
7725	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7726	*/
7727	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7728	{
7729	AssertPtr(pVCpu);
7730	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7731	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7732	}
7733
7734
7735	/**
7736	* Check per-VM and per-VCPU force flag actions that require us to go back to
7737	* ring-3 for one reason or another.
7738	*
7739	* @returns Strict VBox status code (i.e. informational status codes too)
7740	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7741	* ring-3.
7742	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7743	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7744	* interrupts)
7745	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7746	* all EMTs to be in ring-3.
7747	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7748	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7749	* to the EM loop.
7750	*
7751	* @param pVCpu The cross context virtual CPU structure.
7752	* @param fStepping Whether we are single-stepping the guest using the
7753	* hypervisor debugger.
7754	*
7755	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7756	* is no longer in VMX non-root mode.
7757	*/
7758	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7759	{
7760	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7761
7762	/*
7763	* Update pending interrupts into the APIC's IRR.
7764	*/
7765	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7766	APICUpdatePendingInterrupts(pVCpu);
7767
7768	/*
7769	* Anything pending? Should be more likely than not if we're doing a good job.
7770	*/
7771	PVM pVM = pVCpu->CTX_SUFF(pVM);
7772	if ( !fStepping
7773	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7774	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7775	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7776	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7777	return VINF_SUCCESS;
7778
7779	/* Pending PGM C3 sync. */
7780	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7781	{
7782	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7783	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7784	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7785	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7786	if (rcStrict != VINF_SUCCESS)
7787	{
7788	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7789	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7790	return rcStrict;
7791	}
7792	}
7793
7794	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7795	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7796	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7797	{
7798	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7799	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7800	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7801	return rc;
7802	}
7803
7804	/* Pending VM request packets, such as hardware interrupts. */
7805	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7806	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7807	{
7808	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7809	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7810	return VINF_EM_PENDING_REQUEST;
7811	}
7812
7813	/* Pending PGM pool flushes. */
7814	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7815	{
7816	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7817	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7818	return VINF_PGM_POOL_FLUSH_PENDING;
7819	}
7820
7821	/* Pending DMA requests. */
7822	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7823	{
7824	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7825	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7826	return VINF_EM_RAW_TO_R3;
7827	}
7828
7829	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7830	/* Pending nested-guest APIC-write (has highest priority among nested-guest FFs). */
7831	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7832	{
7833	Log4Func(("Pending nested-guest APIC-write\n"));
7834	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
7835	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7836	return rcStrict;
7837	}
7838	/** @todo VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_PREEMPT_TIMER */
7839	#endif
7840
7841	return VINF_SUCCESS;
7842	}
7843
7844
7845	/**
7846	* Converts any TRPM trap into a pending HM event. This is typically used when
7847	* entering from ring-3 (not longjmp returns).
7848	*
7849	* @param pVCpu The cross context virtual CPU structure.
7850	*/
7851	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7852	{
7853	Assert(TRPMHasTrap(pVCpu));
7854	Assert(!pVCpu->hm.s.Event.fPending);
7855
7856	uint8_t uVector;
7857	TRPMEVENT enmTrpmEvent;
7858	RTGCUINT uErrCode;
7859	RTGCUINTPTR GCPtrFaultAddress;
7860	uint8_t cbInstr;
7861
7862	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7863	AssertRC(rc);
7864
7865	uint32_t u32IntInfo;
7866	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
7867	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent);
7868
7869	rc = TRPMResetTrap(pVCpu);
7870	AssertRC(rc);
7871	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7872	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7873
7874	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7875	}
7876
7877
7878	/**
7879	* Converts the pending HM event into a TRPM trap.
7880	*
7881	* @param pVCpu The cross context virtual CPU structure.
7882	*/
7883	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7884	{
7885	Assert(pVCpu->hm.s.Event.fPending);
7886
7887	/* If a trap was already pending, we did something wrong! */
7888	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7889
7890	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
7891	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
7892	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
7893
7894	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7895
7896	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7897	AssertRC(rc);
7898
7899	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7900	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
7901
7902	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
7903	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7904	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
7905	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7906
7907	/* We're now done converting the pending event. */
7908	pVCpu->hm.s.Event.fPending = false;
7909	}
7910
7911
7912	/**
7913	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7914	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7915	*
7916	* @param pVCpu The cross context virtual CPU structure.
7917	* @param pVmcsInfo The VMCS info. object.
7918	*/
7919	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7920	{
7921	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7922	{
7923	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7924	{
7925	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7926	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7927	AssertRC(rc);
7928	}
7929	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7930	}
7931
7932
7933	/**
7934	* Clears the interrupt-window exiting control in the VMCS.
7935	*
7936	* @param pVmcsInfo The VMCS info. object.
7937	*/
7938	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7939	{
7940	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7941	{
7942	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7943	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7944	}
7945	return VINF_SUCCESS;
7946	}
7947
7948
7949	/**
7950	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7951	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7952	*
7953	* @param pVCpu The cross context virtual CPU structure.
7954	* @param pVmcsInfo The VMCS info. object.
7955	*/
7956	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7957	{
7958	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7959	{
7960	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7961	{
7962	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7963	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7964	AssertRC(rc);
7965	Log4Func(("Setup NMI-window exiting\n"));
7966	}
7967	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7968	}
7969
7970
7971	/**
7972	* Clears the NMI-window exiting control in the VMCS.
7973	*
7974	* @param pVmcsInfo The VMCS info. object.
7975	*/
7976	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7977	{
7978	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7979	{
7980	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7981	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7982	}
7983	return VINF_SUCCESS;
7984	}
7985
7986
7987	/**
7988	* Does the necessary state syncing before returning to ring-3 for any reason
7989	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7990	*
7991	* @returns VBox status code.
7992	* @param pVCpu The cross context virtual CPU structure.
7993	* @param fImportState Whether to import the guest state from the VMCS back
7994	* to the guest-CPU context.
7995	*
7996	* @remarks No-long-jmp zone!!!
7997	*/
7998	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7999	{
8000	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8001	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8002
8003	RTCPUID const idCpu = RTMpCpuId();
8004	Log4Func(("HostCpuId=%u\n", idCpu));
8005
8006	/*
8007	* !!! IMPORTANT !!!
8008	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8009	*/
8010
8011	/* Save the guest state if necessary. */
8012	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8013	if (fImportState)
8014	{
8015	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8016	AssertRCReturn(rc, rc);
8017	}
8018
8019	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8020	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8021	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8022
8023	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8024	#ifdef VBOX_STRICT
8025	if (CPUMIsHyperDebugStateActive(pVCpu))
8026	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8027	#endif
8028	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8029	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8030	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8031
8032	/* Restore host-state bits that VT-x only restores partially. */
8033	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8034	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8035	{
8036	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8037	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8038	}
8039	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8040
8041	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8042	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8043	{
8044	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8045	if (!fImportState)
8046	{
8047	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8048	AssertRCReturn(rc, rc);
8049	}
8050	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8051	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8052	}
8053	else
8054	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8055
8056	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8057	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8058
8059	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8060	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8061	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8062	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8063	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8064	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8065	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8066	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8067	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8068	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8069
8070	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8071
8072	/** @todo This partially defeats the purpose of having preemption hooks.
8073	* The problem is, deregistering the hooks should be moved to a place that
8074	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8075	* context.
8076	*/
8077	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8078	AssertRCReturn(rc, rc);
8079
8080	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8081	/*
8082	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8083	* clear a shadow VMCS before allowing that VMCS to become active on another
8084	* logical processor. We may or may not be importing guest state which clears
8085	* it, so cover for it here.
8086	*
8087	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8088	*/
8089	if ( pVmcsInfo->pvShadowVmcs
8090	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8091	{
8092	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8093	AssertRCReturn(rc, rc);
8094	}
8095
8096	/*
8097	* Flag that we need to re-import the host state if we switch to this VMCS before
8098	* executing guest or nested-guest code.
8099	*/
8100	pVmcsInfo->idHostCpu = NIL_RTCPUID;
8101	#endif
8102
8103	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8104	NOREF(idCpu);
8105	return VINF_SUCCESS;
8106	}
8107
8108
8109	/**
8110	* Leaves the VT-x session.
8111	*
8112	* @returns VBox status code.
8113	* @param pVCpu The cross context virtual CPU structure.
8114	*
8115	* @remarks No-long-jmp zone!!!
8116	*/
8117	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8118	{
8119	HM_DISABLE_PREEMPT(pVCpu);
8120	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8121	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8123
8124	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8125	and done this from the VMXR0ThreadCtxCallback(). */
8126	if (!pVCpu->hm.s.fLeaveDone)
8127	{
8128	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8129	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8130	pVCpu->hm.s.fLeaveDone = true;
8131	}
8132	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8133
8134	/*
8135	* !!! IMPORTANT !!!
8136	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8137	*/
8138
8139	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8140	/** @todo Deregistering here means we need to VMCLEAR always
8141	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8142	* for calling VMMR0ThreadCtxHookDisable here! */
8143	VMMR0ThreadCtxHookDisable(pVCpu);
8144
8145	/* Leave HM context. This takes care of local init (term). */
8146	int rc = HMR0LeaveCpu(pVCpu);
8147
8148	HM_RESTORE_PREEMPT();
8149	return rc;
8150	}
8151
8152
8153	/**
8154	* Does the necessary state syncing before doing a longjmp to ring-3.
8155	*
8156	* @returns VBox status code.
8157	* @param pVCpu The cross context virtual CPU structure.
8158	*
8159	* @remarks No-long-jmp zone!!!
8160	*/
8161	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8162	{
8163	return hmR0VmxLeaveSession(pVCpu);
8164	}
8165
8166
8167	/**
8168	* Take necessary actions before going back to ring-3.
8169	*
8170	* An action requires us to go back to ring-3. This function does the necessary
8171	* steps before we can safely return to ring-3. This is not the same as longjmps
8172	* to ring-3, this is voluntary and prepares the guest so it may continue
8173	* executing outside HM (recompiler/IEM).
8174	*
8175	* @returns VBox status code.
8176	* @param pVCpu The cross context virtual CPU structure.
8177	* @param rcExit The reason for exiting to ring-3. Can be
8178	* VINF_VMM_UNKNOWN_RING3_CALL.
8179	*/
8180	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8181	{
8182	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8183
8184	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8185	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8186	{
8187	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8188	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8189	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8190	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8191	}
8192
8193	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8194	VMMRZCallRing3Disable(pVCpu);
8195	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8196
8197	/*
8198	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8199	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8200	*
8201	* This is because execution may continue from ring-3 and we would need to inject
8202	* the event from there (hence place it back in TRPM).
8203	*/
8204	if (pVCpu->hm.s.Event.fPending)
8205	{
8206	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8207	Assert(!pVCpu->hm.s.Event.fPending);
8208
8209	/* Clear the events from the VMCS. */
8210	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8211	AssertRCReturn(rc, rc);
8212	}
8213	#ifdef VBOX_STRICT
8214	else
8215	{
8216	/*
8217	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8218	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8219	* occasionally, see @bugref{9180#c42}.
8220	*
8221	* However, if the VM-entry failed, any VM entry-interruption info. field would
8222	* be left unmodified as the event would not have been injected to the guest. In
8223	* such cases, don't assert, we're not going to continue guest execution anyway.
8224	*/
8225	uint32_t uExitReason;
8226	uint32_t uEntryIntInfo;
8227	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8228	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8229	AssertRC(rc);
8230	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8231	}
8232	#endif
8233
8234	/*
8235	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8236	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8237	* (e.g. TPR below threshold).
8238	*/
8239	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8240	{
8241	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8242	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8243	AssertRCReturn(rc, rc);
8244	}
8245
8246	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8247	and if we're injecting an event we should have a TRPM trap pending. */
8248	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8249	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8250	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8251	#endif
8252
8253	/* Save guest state and restore host state bits. */
8254	int rc = hmR0VmxLeaveSession(pVCpu);
8255	AssertRCReturn(rc, rc);
8256	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8257
8258	/* Thread-context hooks are unregistered at this point!!! */
8259
8260	/* Sync recompiler state. */
8261	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8262	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8263	\| CPUM_CHANGED_LDTR
8264	\| CPUM_CHANGED_GDTR
8265	\| CPUM_CHANGED_IDTR
8266	\| CPUM_CHANGED_TR
8267	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8268	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8269	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8270	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8271
8272	Assert(!pVCpu->hm.s.fClearTrapFlag);
8273
8274	/* Update the exit-to-ring 3 reason. */
8275	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8276
8277	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8278	if ( rcExit != VINF_EM_RAW_INTERRUPT
8279	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8280	{
8281	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8282	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8283	}
8284
8285	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8286
8287	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8288	VMMRZCallRing3RemoveNotification(pVCpu);
8289	VMMRZCallRing3Enable(pVCpu);
8290
8291	return rc;
8292	}
8293
8294
8295	/**
8296	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8297	* longjump to ring-3 and possibly get preempted.
8298	*
8299	* @returns VBox status code.
8300	* @param pVCpu The cross context virtual CPU structure.
8301	* @param enmOperation The operation causing the ring-3 longjump.
8302	* @param pvUser User argument, currently unused, NULL.
8303	*/
8304	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8305	{
8306	RT_NOREF(pvUser);
8307	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8308	{
8309	/*
8310	* !!! IMPORTANT !!!
8311	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8312	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8313	*/
8314	VMMRZCallRing3RemoveNotification(pVCpu);
8315	VMMRZCallRing3Disable(pVCpu);
8316	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8317	RTThreadPreemptDisable(&PreemptState);
8318
8319	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8320	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8321	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8322	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8323
8324	/* Restore host-state bits that VT-x only restores partially. */
8325	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8326	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8327	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8328	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8329
8330	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8331	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8332	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8333
8334	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8335	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8336	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8337
8338	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8339	cleared as part of importing the guest state above. */
8340	hmR0VmxClearVmcs(pVmcsInfo);
8341
8342	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8343	VMMR0ThreadCtxHookDisable(pVCpu);
8344	HMR0LeaveCpu(pVCpu);
8345	RTThreadPreemptRestore(&PreemptState);
8346	return VINF_SUCCESS;
8347	}
8348
8349	Assert(pVCpu);
8350	Assert(pvUser);
8351	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8352	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8353
8354	VMMRZCallRing3Disable(pVCpu);
8355	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8356
8357	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8358
8359	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8360	AssertRCReturn(rc, rc);
8361
8362	VMMRZCallRing3Enable(pVCpu);
8363	return VINF_SUCCESS;
8364	}
8365
8366
8367	/**
8368	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8369	* stack.
8370	*
8371	* @returns Strict VBox status code (i.e. informational status codes too).
8372	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8373	* @param pVCpu The cross context virtual CPU structure.
8374	* @param uValue The value to push to the guest stack.
8375	*/
8376	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8377	{
8378	/*
8379	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8380	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8381	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8382	*/
8383	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8384	if (pCtx->sp == 1)
8385	return VINF_EM_RESET;
8386	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8387	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8388	AssertRC(rc);
8389	return rc;
8390	}
8391
8392
8393	/**
8394	* Injects an event into the guest upon VM-entry by updating the relevant fields
8395	* in the VM-entry area in the VMCS.
8396	*
8397	* @returns Strict VBox status code (i.e. informational status codes too).
8398	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8399	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8400	*
8401	* @param pVCpu The cross context virtual CPU structure.
8402	* @param pVmxTransient The VMX-transient structure.
8403	* @param pEvent The event being injected.
8404	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8405	* will be updated if necessary. This cannot not be NULL.
8406	* @param fStepping Whether we're single-stepping guest execution and should
8407	* return VINF_EM_DBG_STEPPED if the event is injected
8408	* directly (registers modified by us, not by hardware on
8409	* VM-entry).
8410	*/
8411	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8412	uint32_t *pfIntrState)
8413	{
8414	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8415	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8416	Assert(pfIntrState);
8417
8418	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8419	uint32_t u32IntInfo = pEvent->u64IntInfo;
8420	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8421	uint32_t const cbInstr = pEvent->cbInstr;
8422	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8423	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8424	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8425
8426	#ifdef VBOX_STRICT
8427	/*
8428	* Validate the error-code-valid bit for hardware exceptions.
8429	* No error codes for exceptions in real-mode.
8430	*
8431	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8432	*/
8433	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8434	&& !CPUMIsGuestInRealModeEx(pCtx))
8435	{
8436	switch (uVector)
8437	{
8438	case X86_XCPT_PF:
8439	case X86_XCPT_DF:
8440	case X86_XCPT_TS:
8441	case X86_XCPT_NP:
8442	case X86_XCPT_SS:
8443	case X86_XCPT_GP:
8444	case X86_XCPT_AC:
8445	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8446	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8447	RT_FALL_THRU();
8448	default:
8449	break;
8450	}
8451	}
8452
8453	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8454	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8455	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8456	#endif
8457
8458	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8459
8460	/*
8461	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8462	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8463	* interrupt handler in the (real-mode) guest.
8464	*
8465	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8466	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8467	*/
8468	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8469	{
8470	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8471	{
8472	/*
8473	* For CPUs with unrestricted guest execution enabled and with the guest
8474	* in real-mode, we must not set the deliver-error-code bit.
8475	*
8476	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8477	*/
8478	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8479	}
8480	else
8481	{
8482	PVM pVM = pVCpu->CTX_SUFF(pVM);
8483	Assert(PDMVmmDevHeapIsEnabled(pVM));
8484	Assert(pVM->hm.s.vmx.pRealModeTSS);
8485	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8486
8487	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8488	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8489	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8490	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8491	AssertRCReturn(rc2, rc2);
8492
8493	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8494	size_t const cbIdtEntry = sizeof(X86IDTR16);
8495	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8496	{
8497	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8498	if (uVector == X86_XCPT_DF)
8499	return VINF_EM_RESET;
8500
8501	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8502	No error codes for exceptions in real-mode. */
8503	if (uVector == X86_XCPT_GP)
8504	{
8505	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8506	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8507	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8508	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8509	HMEVENT EventXcptDf;
8510	RT_ZERO(EventXcptDf);
8511	EventXcptDf.u64IntInfo = uXcptDfInfo;
8512	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8513	}
8514
8515	/*
8516	* If we're injecting an event with no valid IDT entry, inject a #GP.
8517	* No error codes for exceptions in real-mode.
8518	*
8519	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8520	*/
8521	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8522	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8523	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8524	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8525	HMEVENT EventXcptGp;
8526	RT_ZERO(EventXcptGp);
8527	EventXcptGp.u64IntInfo = uXcptGpInfo;
8528	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8529	}
8530
8531	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8532	uint16_t uGuestIp = pCtx->ip;
8533	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8534	{
8535	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8536	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8537	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8538	}
8539	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8540	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8541
8542	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8543	X86IDTR16 IdtEntry;
8544	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8545	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8546	AssertRCReturn(rc2, rc2);
8547
8548	/* Construct the stack frame for the interrupt/exception handler. */
8549	VBOXSTRICTRC rcStrict;
8550	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8551	if (rcStrict == VINF_SUCCESS)
8552	{
8553	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8554	if (rcStrict == VINF_SUCCESS)
8555	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8556	}
8557
8558	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8559	if (rcStrict == VINF_SUCCESS)
8560	{
8561	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8562	pCtx->rip = IdtEntry.offSel;
8563	pCtx->cs.Sel = IdtEntry.uSel;
8564	pCtx->cs.ValidSel = IdtEntry.uSel;
8565	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8566	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8567	&& uVector == X86_XCPT_PF)
8568	pCtx->cr2 = GCPtrFault;
8569
8570	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8571	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8572	\| HM_CHANGED_GUEST_RSP);
8573
8574	/*
8575	* If we delivered a hardware exception (other than an NMI) and if there was
8576	* block-by-STI in effect, we should clear it.
8577	*/
8578	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8579	{
8580	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8581	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8582	Log4Func(("Clearing inhibition due to STI\n"));
8583	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8584	}
8585
8586	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8587	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8588
8589	/*
8590	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8591	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8592	*/
8593	pVCpu->hm.s.Event.fPending = false;
8594
8595	/*
8596	* If we eventually support nested-guest execution without unrestricted guest execution,
8597	* we should set fInterceptEvents here.
8598	*/
8599	Assert(!pVmxTransient->fIsNestedGuest);
8600
8601	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8602	if (fStepping)
8603	rcStrict = VINF_EM_DBG_STEPPED;
8604	}
8605	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8606	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8607	return rcStrict;
8608	}
8609	}
8610
8611	/*
8612	* Validate.
8613	*/
8614	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8615	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8616
8617	/*
8618	* Inject the event into the VMCS.
8619	*/
8620	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8621	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8622	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8623	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8624	AssertRCReturn(rc, rc);
8625
8626	/*
8627	* Update guest CR2 if this is a page-fault.
8628	*/
8629	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8630	pCtx->cr2 = GCPtrFault;
8631
8632	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8633	return VINF_SUCCESS;
8634	}
8635
8636
8637	/**
8638	* Evaluates the event to be delivered to the guest and sets it as the pending
8639	* event.
8640	*
8641	* @returns Strict VBox status code (i.e. informational status codes too).
8642	* @param pVCpu The cross context virtual CPU structure.
8643	* @param pVmxTransient The VMX-transient structure.
8644	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8645	*/
8646	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8647	{
8648	Assert(pfIntrState);
8649	Assert(!TRPMHasTrap(pVCpu));
8650
8651	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8652	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8653	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8654
8655	/*
8656	* Get the current interruptibility-state of the guest or nested-guest and
8657	* then figure out what needs to be injected.
8658	*/
8659	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8660	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8661	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8662	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8663
8664	/* We don't support block-by-SMI yet.*/
8665	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
8666
8667	/* Block-by-STI must not be set when interrupts are disabled. */
8668	if (fBlockSti)
8669	{
8670	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
8671	Assert(pCtx->eflags.Bits.u1IF);
8672	}
8673
8674	/* Update interruptibility state to the caller. */
8675	*pfIntrState = fIntrState;
8676
8677	/*
8678	* Toggling of interrupt force-flags here is safe since we update TRPM on
8679	* premature exits to ring-3 before executing guest code, see hmR0VmxExitToRing3().
8680	* We must NOT restore these force-flags.
8681	*/
8682
8683	/** @todo SMI. SMIs take priority over NMIs. */
8684
8685	/*
8686	* Check if an NMI is pending and if the guest or nested-guest can receive them.
8687	* NMIs take priority over external interrupts.
8688	*/
8689	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8690	{
8691	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8692	if ( !pVCpu->hm.s.Event.fPending
8693	&& !fBlockNmi
8694	&& !fBlockSti
8695	&& !fBlockMovSS)
8696	{
8697	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8698	if ( fIsNestedGuest
8699	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8700	return IEMExecVmxVmexitXcptNmi(pVCpu);
8701	#endif
8702	hmR0VmxSetPendingXcptNmi(pVCpu);
8703	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8704	Log4Func(("Pending NMI\n"));
8705	}
8706	else if (!fIsNestedGuest)
8707	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8708	/* else: for nested-guests, NMI-window exiting will be picked up when merging VMCS controls. */
8709	}
8710	/*
8711	* Check if an external interrupt (PIC/APIC) is pending and if the guest or nested-guest
8712	* can receive them. Once PDMGetInterrupt() returns a valid interrupt we -must- deliver
8713	* the interrupt. We can no longer re-request it from the APIC.
8714	*/
8715	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8716	&& !pVCpu->hm.s.fSingleInstruction)
8717	{
8718	Assert(!DBGFIsStepping(pVCpu));
8719	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8720	AssertRCReturn(rc, rc);
8721
8722	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8723	if ( !pVCpu->hm.s.Event.fPending
8724	&& !fBlockInt
8725	&& !fBlockSti
8726	&& !fBlockMovSS)
8727	{
8728	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8729	if ( fIsNestedGuest
8730	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8731	&& !CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8732	{
8733	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8734	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8735	return rcStrict;
8736	}
8737	#endif
8738	uint8_t u8Interrupt;
8739	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8740	if (RT_SUCCESS(rc))
8741	{
8742	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8743	if ( fIsNestedGuest
8744	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8745	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8746	{
8747	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8748	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8749	return rcStrict;
8750	}
8751	#endif
8752	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8753	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8754	}
8755	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8756	{
8757	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8758
8759	if ( !fIsNestedGuest
8760	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8761	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8762	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8763
8764	/*
8765	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8766	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8767	* need to re-set this force-flag here.
8768	*/
8769	}
8770	else
8771	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8772	}
8773	else if (!fIsNestedGuest)
8774	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8775	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCS controls. */
8776	}
8777
8778	return VINF_SUCCESS;
8779	}
8780
8781
8782	/**
8783	* Injects any pending events into the guest if the guest is in a state to
8784	* receive them.
8785	*
8786	* @returns Strict VBox status code (i.e. informational status codes too).
8787	* @param pVCpu The cross context virtual CPU structure.
8788	* @param pVmxTransient The VMX-transient structure.
8789	* @param fIntrState The VT-x guest-interruptibility state.
8790	* @param fStepping Whether we are single-stepping the guest using the
8791	* hypervisor debugger and should return
8792	* VINF_EM_DBG_STEPPED if the event was dispatched
8793	* directly.
8794	*/
8795	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8796	{
8797	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8798	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8799
8800	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8801	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8802
8803	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8804	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8805	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8806	Assert(!TRPMHasTrap(pVCpu));
8807
8808	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8809	if (pVCpu->hm.s.Event.fPending)
8810	{
8811	/*
8812	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8813	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8814	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8815	*
8816	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8817	*/
8818	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8819	#ifdef VBOX_STRICT
8820	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8821	{
8822	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8823	Assert(!fBlockInt);
8824	Assert(!fBlockSti);
8825	Assert(!fBlockMovSS);
8826	}
8827	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8828	{
8829	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8830	Assert(!fBlockSti);
8831	Assert(!fBlockMovSS);
8832	Assert(!fBlockNmi);
8833	}
8834	#endif
8835	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8836	uIntType));
8837
8838	/*
8839	* Inject the event and get any changes to the guest-interruptibility state.
8840	*
8841	* The guest-interruptibility state may need to be updated if we inject the event
8842	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8843	*/
8844	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8845	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8846
8847	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8848	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8849	else
8850	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8851	}
8852
8853	/*
8854	* Update the guest-interruptibility state.
8855	*
8856	* This is required for the real-on-v86 software interrupt injection case above, as well as
8857	* updates to the guest state from ring-3 or IEM/REM.
8858	*/
8859	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8860	AssertRCReturn(rc, rc);
8861
8862	/*
8863	* There's no need to clear the VM-entry interruption-information field here if we're not
8864	* injecting anything. VT-x clears the valid bit on every VM-exit.
8865	*
8866	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8867	*/
8868
8869	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8870	NOREF(fBlockMovSS); NOREF(fBlockSti);
8871	return rcStrict;
8872	}
8873
8874
8875	/**
8876	* Enters the VT-x session.
8877	*
8878	* @returns VBox status code.
8879	* @param pVCpu The cross context virtual CPU structure.
8880	*/
8881	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8882	{
8883	AssertPtr(pVCpu);
8884	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8885	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8886
8887	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8888	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8889	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8890
8891	#ifdef VBOX_STRICT
8892	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8893	RTCCUINTREG uHostCr4 = ASMGetCR4();
8894	if (!(uHostCr4 & X86_CR4_VMXE))
8895	{
8896	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8897	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8898	}
8899	#endif
8900
8901	/*
8902	* Load the appropriate VMCS as the current and active one.
8903	*/
8904	PVMXVMCSINFO pVmcsInfo;
8905	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8906	if (!fInNestedGuestMode)
8907	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8908	else
8909	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8910	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8911	if (RT_SUCCESS(rc))
8912	{
8913	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8914	pVCpu->hm.s.fLeaveDone = false;
8915	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8916
8917	/*
8918	* Do the EMT scheduled L1D flush here if needed.
8919	*/
8920	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8921	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8922	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8923	hmR0MdsClear();
8924	}
8925	return rc;
8926	}
8927
8928
8929	/**
8930	* The thread-context callback (only on platforms which support it).
8931	*
8932	* @param enmEvent The thread-context event.
8933	* @param pVCpu The cross context virtual CPU structure.
8934	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8935	* @thread EMT(pVCpu)
8936	*/
8937	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8938	{
8939	AssertPtr(pVCpu);
8940	RT_NOREF1(fGlobalInit);
8941
8942	switch (enmEvent)
8943	{
8944	case RTTHREADCTXEVENT_OUT:
8945	{
8946	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8947	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8948	VMCPU_ASSERT_EMT(pVCpu);
8949
8950	/* No longjmps (logger flushes, locks) in this fragile context. */
8951	VMMRZCallRing3Disable(pVCpu);
8952	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8953
8954	/* Restore host-state (FPU, debug etc.) */
8955	if (!pVCpu->hm.s.fLeaveDone)
8956	{
8957	/*
8958	* Do -not- import the guest-state here as we might already be in the middle of importing
8959	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8960	*/
8961	hmR0VmxLeave(pVCpu, false /* fImportState */);
8962	pVCpu->hm.s.fLeaveDone = true;
8963	}
8964
8965	/* Leave HM context, takes care of local init (term). */
8966	int rc = HMR0LeaveCpu(pVCpu);
8967	AssertRC(rc);
8968
8969	/* Restore longjmp state. */
8970	VMMRZCallRing3Enable(pVCpu);
8971	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8972	break;
8973	}
8974
8975	case RTTHREADCTXEVENT_IN:
8976	{
8977	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8978	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8979	VMCPU_ASSERT_EMT(pVCpu);
8980
8981	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8982	VMMRZCallRing3Disable(pVCpu);
8983	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8984
8985	/* Initialize the bare minimum state required for HM. This takes care of
8986	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8987	int rc = hmR0EnterCpu(pVCpu);
8988	AssertRC(rc);
8989	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8990	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8991
8992	/* Load the active VMCS as the current one. */
8993	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8994	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8995	AssertRC(rc);
8996	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8997	pVCpu->hm.s.fLeaveDone = false;
8998
8999	/* Do the EMT scheduled L1D flush if needed. */
9000	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9001	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9002
9003	/* Restore longjmp state. */
9004	VMMRZCallRing3Enable(pVCpu);
9005	break;
9006	}
9007
9008	default:
9009	break;
9010	}
9011	}
9012
9013
9014	/**
9015	* Exports the host state into the VMCS host-state area.
9016	* Sets up the VM-exit MSR-load area.
9017	*
9018	* The CPU state will be loaded from these fields on every successful VM-exit.
9019	*
9020	* @returns VBox status code.
9021	* @param pVCpu The cross context virtual CPU structure.
9022	*
9023	* @remarks No-long-jump zone!!!
9024	*/
9025	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9026	{
9027	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9028
9029	int rc = VINF_SUCCESS;
9030	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9031	{
9032	rc = hmR0VmxExportHostControlRegs();
9033	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9034
9035	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9036	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9037
9038	rc = hmR0VmxExportHostMsrs(pVCpu);
9039	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9040
9041	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9042	}
9043	return rc;
9044	}
9045
9046
9047	/**
9048	* Saves the host state in the VMCS host-state.
9049	*
9050	* @returns VBox status code.
9051	* @param pVCpu The cross context virtual CPU structure.
9052	*
9053	* @remarks No-long-jump zone!!!
9054	*/
9055	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9056	{
9057	AssertPtr(pVCpu);
9058	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9059
9060	/*
9061	* Export the host state here while entering HM context.
9062	* When thread-context hooks are used, we might get preempted and have to re-save the host
9063	* state but most of the time we won't be, so do it here before we disable interrupts.
9064	*/
9065	return hmR0VmxExportHostState(pVCpu);
9066	}
9067
9068
9069	/**
9070	* Exports the guest state into the VMCS guest-state area.
9071	*
9072	* The will typically be done before VM-entry when the guest-CPU state and the
9073	* VMCS state may potentially be out of sync.
9074	*
9075	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9076	* VM-entry controls.
9077	* Sets up the appropriate VMX non-root function to execute guest code based on
9078	* the guest CPU mode.
9079	*
9080	* @returns VBox strict status code.
9081	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9082	* without unrestricted guest execution and the VMMDev is not presently
9083	* mapped (e.g. EFI32).
9084	*
9085	* @param pVCpu The cross context virtual CPU structure.
9086	* @param pVmxTransient The VMX-transient structure.
9087	*
9088	* @remarks No-long-jump zone!!!
9089	*/
9090	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9091	{
9092	AssertPtr(pVCpu);
9093	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9094	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9095
9096	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9097
9098	/*
9099	* Determine real-on-v86 mode.
9100	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9101	*/
9102	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9103	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9104	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9105	pVmcsInfo->RealMode. fRealOnV86Active = false;
9106	else
9107	{
9108	Assert(!pVmxTransient->fIsNestedGuest);
9109	pVmcsInfo->RealMode.fRealOnV86Active = true;
9110	}
9111
9112	/*
9113	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9114	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9115	*/
9116	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9117	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9118	* be a need to evaluate this everytime since I'm pretty sure we intercept
9119	* all guest paging mode changes. */
9120	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9121	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9122
9123	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9124	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9125
9126	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9127	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9128
9129	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9130	if (rcStrict == VINF_SUCCESS)
9131	{ /* likely */ }
9132	else
9133	{
9134	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9135	return rcStrict;
9136	}
9137
9138	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9139	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9140
9141	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9142	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9143
9144	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9145	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9146
9147	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9148	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9149
9150	rc = hmR0VmxExportGuestRip(pVCpu);
9151	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9152	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9153	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9154
9155	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9156	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9157
9158	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9159	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9160	\| HM_CHANGED_GUEST_CR2
9161	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9162	\| HM_CHANGED_GUEST_X87
9163	\| HM_CHANGED_GUEST_SSE_AVX
9164	\| HM_CHANGED_GUEST_OTHER_XSAVE
9165	\| HM_CHANGED_GUEST_XCRx
9166	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9167	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9168	\| HM_CHANGED_GUEST_TSC_AUX
9169	\| HM_CHANGED_GUEST_OTHER_MSRS
9170	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9171
9172	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9173	return rc;
9174	}
9175
9176
9177	/**
9178	* Exports the state shared between the host and guest into the VMCS.
9179	*
9180	* @param pVCpu The cross context virtual CPU structure.
9181	* @param pVmxTransient The VMX-transient structure.
9182	*
9183	* @remarks No-long-jump zone!!!
9184	*/
9185	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9186	{
9187	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9188	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9189
9190	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9191	{
9192	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9193	AssertRC(rc);
9194	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9195
9196	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9197	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9198	{
9199	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9200	AssertRC(rc);
9201	}
9202	}
9203
9204	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9205	{
9206	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9207	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9208	}
9209
9210	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9211	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9212	}
9213
9214
9215	/**
9216	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9217	*
9218	* @returns Strict VBox status code (i.e. informational status codes too).
9219	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9220	* without unrestricted guest execution and the VMMDev is not presently
9221	* mapped (e.g. EFI32).
9222	*
9223	* @param pVCpu The cross context virtual CPU structure.
9224	* @param pVmxTransient The VMX-transient structure.
9225	*
9226	* @remarks No-long-jump zone!!!
9227	*/
9228	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9229	{
9230	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9231	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9232	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9233
9234	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9235	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9236	#endif
9237
9238	/*
9239	* For many exits it's only RIP that changes and hence try to export it first
9240	* without going through a lot of change flag checks.
9241	*/
9242	VBOXSTRICTRC rcStrict;
9243	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9244	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9245	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9246	{
9247	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9248	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9249	{ /* likely */}
9250	else
9251	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9252	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9253	}
9254	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9255	{
9256	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9257	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9258	{ /* likely */}
9259	else
9260	{
9261	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9262	VBOXSTRICTRC_VAL(rcStrict)));
9263	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9264	return rcStrict;
9265	}
9266	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9267	}
9268	else
9269	rcStrict = VINF_SUCCESS;
9270
9271	#ifdef VBOX_STRICT
9272	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9273	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9274	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9275	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9276	("fCtxChanged=%#RX64\n", fCtxChanged));
9277	#endif
9278	return rcStrict;
9279	}
9280
9281
9282	/**
9283	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9284	* and update error record fields accordingly.
9285	*
9286	* @returns VMX_IGS_* error codes.
9287	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9288	* wrong with the guest state.
9289	*
9290	* @param pVCpu The cross context virtual CPU structure.
9291	* @param pVmcsInfo The VMCS info. object.
9292	*
9293	* @remarks This function assumes our cache of the VMCS controls
9294	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9295	*/
9296	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9297	{
9298	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9299	#define HMVMX_CHECK_BREAK(expr, err) do { \
9300	if (!(expr)) { uError = (err); break; } \
9301	} while (0)
9302
9303	int rc;
9304	PVM pVM = pVCpu->CTX_SUFF(pVM);
9305	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9306	uint32_t uError = VMX_IGS_ERROR;
9307	uint32_t u32Val;
9308	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9309
9310	do
9311	{
9312	/*
9313	* CR0.
9314	*/
9315	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9316	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9317	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9318	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9319	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9320	if (fUnrestrictedGuest)
9321	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9322
9323	uint32_t u32GuestCr0;
9324	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9325	AssertRCBreak(rc);
9326	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9327	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9328	if ( !fUnrestrictedGuest
9329	&& (u32GuestCr0 & X86_CR0_PG)
9330	&& !(u32GuestCr0 & X86_CR0_PE))
9331	{
9332	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9333	}
9334
9335	/*
9336	* CR4.
9337	*/
9338	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9339	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9340	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9341
9342	uint32_t u32GuestCr4;
9343	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9344	AssertRCBreak(rc);
9345	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9346	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9347
9348	/*
9349	* IA32_DEBUGCTL MSR.
9350	*/
9351	uint64_t u64Val;
9352	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9353	AssertRCBreak(rc);
9354	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9355	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9356	{
9357	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9358	}
9359	uint64_t u64DebugCtlMsr = u64Val;
9360
9361	#ifdef VBOX_STRICT
9362	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9363	AssertRCBreak(rc);
9364	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9365	#endif
9366	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9367
9368	/*
9369	* RIP and RFLAGS.
9370	*/
9371	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
9372	AssertRCBreak(rc);
9373	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9374	if ( !fLongModeGuest
9375	\|\| !pCtx->cs.Attr.n.u1Long)
9376	{
9377	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9378	}
9379	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9380	* must be identical if the "IA-32e mode guest" VM-entry
9381	* control is 1 and CS.L is 1. No check applies if the
9382	* CPU supports 64 linear-address bits. */
9383
9384	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9385	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9386	AssertRCBreak(rc);
9387	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9388	VMX_IGS_RFLAGS_RESERVED);
9389	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9390	uint32_t const u32Eflags = u64Val;
9391
9392	if ( fLongModeGuest
9393	\|\| ( fUnrestrictedGuest
9394	&& !(u32GuestCr0 & X86_CR0_PE)))
9395	{
9396	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9397	}
9398
9399	uint32_t u32EntryInfo;
9400	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9401	AssertRCBreak(rc);
9402	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9403	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9404
9405	/*
9406	* 64-bit checks.
9407	*/
9408	if (fLongModeGuest)
9409	{
9410	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9411	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9412	}
9413
9414	if ( !fLongModeGuest
9415	&& (u32GuestCr4 & X86_CR4_PCIDE))
9416	{
9417	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9418	}
9419
9420	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9421	* 51:32 beyond the processor's physical-address width are 0. */
9422
9423	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9424	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9425	{
9426	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9427	}
9428
9429	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9430	AssertRCBreak(rc);
9431	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9432
9433	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9434	AssertRCBreak(rc);
9435	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9436
9437	/*
9438	* PERF_GLOBAL MSR.
9439	*/
9440	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9441	{
9442	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9443	AssertRCBreak(rc);
9444	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9445	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9446	}
9447
9448	/*
9449	* PAT MSR.
9450	*/
9451	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9452	{
9453	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9454	AssertRCBreak(rc);
9455	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9456	for (unsigned i = 0; i < 8; i++)
9457	{
9458	uint8_t u8Val = (u64Val & 0xff);
9459	if ( u8Val != 0 /* UC */
9460	&& u8Val != 1 /* WC */
9461	&& u8Val != 4 /* WT */
9462	&& u8Val != 5 /* WP */
9463	&& u8Val != 6 /* WB */
9464	&& u8Val != 7 /* UC- */)
9465	{
9466	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9467	}
9468	u64Val >>= 8;
9469	}
9470	}
9471
9472	/*
9473	* EFER MSR.
9474	*/
9475	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9476	{
9477	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9478	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9479	AssertRCBreak(rc);
9480	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9481	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9482	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9483	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9484	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9485	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9486	* iemVmxVmentryCheckGuestState(). */
9487	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9488	\|\| !(u32GuestCr0 & X86_CR0_PG)
9489	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9490	VMX_IGS_EFER_LMA_LME_MISMATCH);
9491	}
9492
9493	/*
9494	* Segment registers.
9495	*/
9496	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9497	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9498	if (!(u32Eflags & X86_EFL_VM))
9499	{
9500	/* CS */
9501	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9502	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9503	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9504	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9505	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9506	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9507	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9508	/* CS cannot be loaded with NULL in protected mode. */
9509	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9510	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9511	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9512	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9513	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9514	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9515	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9516	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9517	else
9518	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9519
9520	/* SS */
9521	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9522	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9523	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9524	if ( !(pCtx->cr0 & X86_CR0_PE)
9525	\|\| pCtx->cs.Attr.n.u4Type == 3)
9526	{
9527	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9528	}
9529	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9530	{
9531	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9532	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9533	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9534	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9535	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9536	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9537	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9538	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9539	}
9540
9541	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9542	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9543	{
9544	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9545	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9546	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9547	\|\| pCtx->ds.Attr.n.u4Type > 11
9548	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9549	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9550	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9551	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9552	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9553	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9554	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9555	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9556	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9557	}
9558	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9559	{
9560	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9561	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9562	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9563	\|\| pCtx->es.Attr.n.u4Type > 11
9564	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9565	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9566	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9567	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9568	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9569	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9570	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9571	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9572	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9573	}
9574	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9575	{
9576	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9577	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9578	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9579	\|\| pCtx->fs.Attr.n.u4Type > 11
9580	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9581	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9582	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9583	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9584	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9585	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9586	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9587	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9588	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9589	}
9590	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9591	{
9592	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9593	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9594	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9595	\|\| pCtx->gs.Attr.n.u4Type > 11
9596	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9597	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9598	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9599	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9600	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9601	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9602	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9603	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9604	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9605	}
9606	/* 64-bit capable CPUs. */
9607	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9608	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9609	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9610	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9611	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9612	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9613	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9614	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9615	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9616	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9617	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9618	}
9619	else
9620	{
9621	/* V86 mode checks. */
9622	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9623	if (pVmcsInfo->RealMode.fRealOnV86Active)
9624	{
9625	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9626	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9627	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9628	}
9629	else
9630	{
9631	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9632	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9633	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9634	}
9635
9636	/* CS */
9637	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9638	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9639	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9640	/* SS */
9641	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9642	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9643	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9644	/* DS */
9645	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9646	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9647	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9648	/* ES */
9649	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9650	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9651	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9652	/* FS */
9653	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9654	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9655	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9656	/* GS */
9657	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9658	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9659	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9660	/* 64-bit capable CPUs. */
9661	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9662	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9663	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9664	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9665	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9666	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9667	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9668	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9669	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9670	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9671	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9672	}
9673
9674	/*
9675	* TR.
9676	*/
9677	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9678	/* 64-bit capable CPUs. */
9679	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9680	if (fLongModeGuest)
9681	{
9682	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9683	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9684	}
9685	else
9686	{
9687	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9688	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9689	VMX_IGS_TR_ATTR_TYPE_INVALID);
9690	}
9691	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9692	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9693	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9694	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9695	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9696	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9697	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9698	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9699
9700	/*
9701	* GDTR and IDTR (64-bit capable checks).
9702	*/
9703	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9704	AssertRCBreak(rc);
9705	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9706
9707	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9708	AssertRCBreak(rc);
9709	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9710
9711	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9712	AssertRCBreak(rc);
9713	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9714
9715	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9716	AssertRCBreak(rc);
9717	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9718
9719	/*
9720	* Guest Non-Register State.
9721	*/
9722	/* Activity State. */
9723	uint32_t u32ActivityState;
9724	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9725	AssertRCBreak(rc);
9726	HMVMX_CHECK_BREAK( !u32ActivityState
9727	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9728	VMX_IGS_ACTIVITY_STATE_INVALID);
9729	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9730	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9731	uint32_t u32IntrState;
9732	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9733	AssertRCBreak(rc);
9734	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9735	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9736	{
9737	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9738	}
9739
9740	/** @todo Activity state and injecting interrupts. Left as a todo since we
9741	* currently don't use activity states but ACTIVE. */
9742
9743	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9744	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9745
9746	/* Guest interruptibility-state. */
9747	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9748	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9749	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9750	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9751	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9752	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9753	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9754	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9755	{
9756	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9757	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9758	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9759	}
9760	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9761	{
9762	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9763	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9764	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9765	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9766	}
9767	/** @todo Assumes the processor is not in SMM. */
9768	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9769	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9770	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9771	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9772	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9773	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9774	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9775	{
9776	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9777	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9778	}
9779
9780	/* Pending debug exceptions. */
9781	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9782	AssertRCBreak(rc);
9783	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9784	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9785	u32Val = u64Val; /* For pending debug exceptions checks below. */
9786
9787	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9788	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9789	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9790	{
9791	if ( (u32Eflags & X86_EFL_TF)
9792	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9793	{
9794	/* Bit 14 is PendingDebug.BS. */
9795	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9796	}
9797	if ( !(u32Eflags & X86_EFL_TF)
9798	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9799	{
9800	/* Bit 14 is PendingDebug.BS. */
9801	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9802	}
9803	}
9804
9805	/* VMCS link pointer. */
9806	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9807	AssertRCBreak(rc);
9808	if (u64Val != UINT64_C(0xffffffffffffffff))
9809	{
9810	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9811	/** @todo Bits beyond the processor's physical-address width MBZ. */
9812	/** @todo SMM checks. */
9813	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
9814	Assert(pVmcsInfo->pvShadowVmcs);
9815	VMXVMCSREVID VmcsRevId;
9816	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
9817	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
9818	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
9819	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
9820	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
9821	}
9822
9823	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9824	* not using nested paging? */
9825	if ( pVM->hm.s.fNestedPaging
9826	&& !fLongModeGuest
9827	&& CPUMIsGuestInPAEModeEx(pCtx))
9828	{
9829	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9830	AssertRCBreak(rc);
9831	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9832
9833	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9834	AssertRCBreak(rc);
9835	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9836
9837	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9838	AssertRCBreak(rc);
9839	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9840
9841	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9842	AssertRCBreak(rc);
9843	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9844	}
9845
9846	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9847	if (uError == VMX_IGS_ERROR)
9848	uError = VMX_IGS_REASON_NOT_FOUND;
9849	} while (0);
9850
9851	pVCpu->hm.s.u32HMError = uError;
9852	return uError;
9853
9854	#undef HMVMX_ERROR_BREAK
9855	#undef HMVMX_CHECK_BREAK
9856	}
9857
9858
9859	/**
9860	* Map the APIC-access page for virtualizing APIC accesses.
9861	*
9862	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
9863	* this not done as part of exporting guest state, see @bugref{8721}.
9864	*
9865	* @returns VBox status code.
9866	* @param pVCpu The cross context virtual CPU structure.
9867	*/
9868	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9869	{
9870	PVM pVM = pVCpu->CTX_SUFF(pVM);
9871	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9872
9873	Assert(PDMHasApic(pVM));
9874	Assert(u64MsrApicBase);
9875
9876	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9877	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9878
9879	/* Unalias the existing mapping. */
9880	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9881	AssertRCReturn(rc, rc);
9882
9883	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9884	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9885	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9886	AssertRCReturn(rc, rc);
9887
9888	/* Update the per-VCPU cache of the APIC base MSR. */
9889	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9890	return VINF_SUCCESS;
9891	}
9892
9893
9894	/**
9895	* Worker function passed to RTMpOnSpecific() that is to be called on the target
9896	* CPU.
9897	*
9898	* @param idCpu The ID for the CPU the function is called on.
9899	* @param pvUser1 Null, not used.
9900	* @param pvUser2 Null, not used.
9901	*/
9902	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
9903	{
9904	RT_NOREF3(idCpu, pvUser1, pvUser2);
9905	VMXDispatchHostNmi();
9906	}
9907
9908
9909	/**
9910	* Dispatching an NMI on the host CPU that received it.
9911	*
9912	* @returns VBox status code.
9913	* @param pVCpu The cross context virtual CPU structure.
9914	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
9915	* executing when receiving the host NMI in VMX non-root
9916	* operation.
9917	*/
9918	static int hmR0VmxExitHostNmi(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9919	{
9920	RTCPUID const idCpu = pVmcsInfo->idHostCpu;
9921
9922	/*
9923	* We don't want to delay dispatching the NMI any more than we have to. However,
9924	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
9925	* after executing guest or nested-guest code for the following reasons:
9926	*
9927	* - We would need to perform VMREADs with interrupts disabled and is orders of
9928	* magnitude worse when we run as a guest hypervisor without VMCS shadowing
9929	* supported by the host hypervisor.
9930	*
9931	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
9932	* longer period of time just for handling an edge case like host NMIs which do
9933	* not occur nearly as frequently as other VM-exits.
9934	*
9935	* Let's cover the most likely scenario first. Check if we are on the target CPU
9936	* and dispatch the NMI right away. This should be much faster than calling into
9937	* RTMpOnSpecific() machinery.
9938	*/
9939	bool fDispatched = false;
9940	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
9941	if (idCpu == RTMpCpuId())
9942	{
9943	VMXDispatchHostNmi();
9944	fDispatched = true;
9945	}
9946	ASMSetFlags(fEFlags);
9947	if (fDispatched)
9948	{
9949	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9950	return VINF_SUCCESS;
9951	}
9952
9953	/*
9954	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
9955	* there should be no race or recursion even if we are unlucky enough to be preempted
9956	* (to the target CPU) without dispatching the host NMI above.
9957	*/
9958	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
9959	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
9960	}
9961
9962
9963	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9964	/**
9965	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9966	* nested-guest using hardware-assisted VMX.
9967	*
9968	* @param pVCpu The cross context virtual CPU structure.
9969	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9970	* @param pVmcsInfoGst The guest VMCS info. object.
9971	*/
9972	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9973	{
9974	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
9975	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9976	Assert(pu64MsrBitmap);
9977
9978	/*
9979	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9980	* MSR that is intercepted by the guest is also intercepted while executing the
9981	* nested-guest using hardware-assisted VMX.
9982	*
9983	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
9984	* nested-guest VM-exit even if the outer guest is not intercepting some
9985	* MSRs. We cannot assume the caller has initialized the nested-guest
9986	* MSR bitmap in this case.
9987	*
9988	* The guest hypervisor may also switch whether it uses MSR bitmaps for
9989	* each VM-entry, hence initializing it once per-VM while setting up the
9990	* nested-guest VMCS is not sufficient.
9991	*/
9992	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9993	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
9994	{
9995	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9996	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9997	Assert(pu64MsrBitmapNstGst);
9998	Assert(pu64MsrBitmapGst);
9999
10000	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10001	for (uint32_t i = 0; i < cFrags; i++)
10002	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10003	}
10004	else
10005	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10006	}
10007
10008
10009	/**
10010	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10011	* hardware-assisted VMX execution of the nested-guest.
10012	*
10013	* For a guest, we don't modify these controls once we set up the VMCS and hence
10014	* this function is never called.
10015	*
10016	* For nested-guests since the guest hypervisor provides these controls on every
10017	* nested-guest VM-entry and could potentially change them everytime we need to
10018	* merge them before every nested-guest VM-entry.
10019	*
10020	* @returns VBox status code.
10021	* @param pVCpu The cross context virtual CPU structure.
10022	*/
10023	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10024	{
10025	PVM pVM = pVCpu->CTX_SUFF(pVM);
10026	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10027	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10028	Assert(pVmcsNstGst);
10029
10030	/*
10031	* Merge the controls with the requirements of the guest VMCS.
10032	*
10033	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10034	* VMCS with the features supported by the physical CPU as it's already done by the
10035	* VMLAUNCH/VMRESUME instruction emulation.
10036	*
10037	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10038	* derived from the VMX features supported by the physical CPU.
10039	*/
10040
10041	/* Pin-based VM-execution controls. */
10042	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10043
10044	/* Processor-based VM-execution controls. */
10045	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10046	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10047	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10048	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10049	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10050
10051	/* Secondary processor-based VM-execution controls. */
10052	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10053	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10054	\| VMX_PROC_CTLS2_INVPCID
10055	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10056	\| VMX_PROC_CTLS2_RDTSCP
10057	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10058	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10059	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10060	\| VMX_PROC_CTLS2_VMFUNC));
10061
10062	/*
10063	* VM-entry controls:
10064	* These controls contains state that depends on the nested-guest state (primarily
10065	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10066	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10067	* properly continue executing the nested-guest if the EFER MSR changes but does not
10068	* cause a nested-guest VM-exits.
10069	*
10070	* VM-exit controls:
10071	* These controls specify the host state on return. We cannot use the controls from
10072	* the guest hypervisor state as is as it would contain the guest state rather than
10073	* the host state. Since the host state is subject to change (e.g. preemption, trips
10074	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10075	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10076	*
10077	* VM-entry MSR-load:
10078	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10079	* context by the VMLAUNCH/VMRESUME instruction emulation.
10080	*
10081	* VM-exit MSR-store:
10082	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10083	* back into the VM-exit MSR-store area.
10084	*
10085	* VM-exit MSR-load areas:
10086	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10087	* can entirely ignore what the guest hypervisor wants to load here.
10088	*/
10089
10090	/*
10091	* Exception bitmap.
10092	*
10093	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10094	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10095	* code more flexible if intercepting exceptions become more dynamic in the future we do
10096	* it as part of exporting the nested-guest state.
10097	*/
10098	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10099
10100	/*
10101	* CR0/CR4 guest/host mask.
10102	*
10103	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10104	* cause VM-exits, so we need to merge them here.
10105	*/
10106	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10107	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10108
10109	/*
10110	* Page-fault error-code mask and match.
10111	*
10112	* Although we require unrestricted guest execution (and thereby nested-paging) for
10113	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10114	* normally intercept #PFs, it might intercept them for debugging purposes.
10115	*
10116	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10117	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10118	*/
10119	uint32_t u32XcptPFMask;
10120	uint32_t u32XcptPFMatch;
10121	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10122	{
10123	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10124	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10125	}
10126	else
10127	{
10128	u32XcptPFMask = 0;
10129	u32XcptPFMatch = 0;
10130	}
10131
10132	/*
10133	* Pause-Loop exiting.
10134	*/
10135	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10136	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10137
10138	/*
10139	* I/O Bitmap.
10140	*
10141	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we always
10142	* intercept all I/O port accesses.
10143	*/
10144	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10145
10146	/*
10147	* VMCS shadowing.
10148	*
10149	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10150	* enabled while executing the nested-guest.
10151	*/
10152	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10153
10154	/*
10155	* APIC-access page.
10156	*/
10157	RTHCPHYS HCPhysApicAccess;
10158	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10159	{
10160	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10161	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10162
10163	/** @todo NSTVMX: This is not really correct but currently is required to make
10164	* things work. We need to re-register the page handler when we fallback to
10165	* IEM execution of the nested-guest! */
10166	PGMHandlerPhysicalDeregister(pVM, GCPhysApicAccess);
10167
10168	void *pvPage;
10169	PGMPAGEMAPLOCK PgMapLockApicAccess;
10170	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgMapLockApicAccess);
10171	if (RT_SUCCESS(rc))
10172	{
10173	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10174	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10175
10176	/*
10177	* We can release the page lock here because the APIC-access page is never read or
10178	* written to but merely serves as a placeholder in the shadow/nested page tables
10179	* to cause VM-exits or re-direct the access to the virtual-APIC page.
10180	*/
10181	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgMapLockApicAccess);
10182	}
10183	else
10184	return rc;
10185	}
10186	else
10187	HCPhysApicAccess = 0;
10188
10189	/*
10190	* Virtual-APIC page and TPR threshold.
10191	*/
10192	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10193	RTHCPHYS HCPhysVirtApic;
10194	uint32_t u32TprThreshold;
10195	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10196	{
10197	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10198
10199	void *pvPage;
10200	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10201	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &pVCpu->hm.s.vmx.PgMapLockVirtApic);
10202	AssertMsgRCReturn(rc, ("Failed to get current-context pointer for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10203
10204	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10205	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10206	pVCpu->hm.s.vmx.fVirtApicPageLocked = true;
10207
10208	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10209	}
10210	else
10211	{
10212	HCPhysVirtApic = 0;
10213	u32TprThreshold = 0;
10214
10215	/*
10216	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10217	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10218	* be taken care of by EPT/shadow paging.
10219	*/
10220	if (pVM->hm.s.fAllow64BitGuests)
10221	{
10222	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10223	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10224	}
10225	}
10226
10227	/*
10228	* Validate basic assumptions.
10229	*/
10230	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10231	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10232	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10233
10234	/*
10235	* Commit it to the nested-guest VMCS.
10236	*/
10237	int rc = VINF_SUCCESS;
10238	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10239	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10240	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10241	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10242	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10243	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10244	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10245	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10246	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10247	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10248	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10249	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10250	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10251	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10252	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10253	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10254	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10255	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10256	{
10257	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10258	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10259	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10260	}
10261	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10262	{
10263	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10264	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10265	}
10266	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10267	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10268	AssertRCReturn(rc, rc);
10269
10270	/*
10271	* Update the nested-guest VMCS cache.
10272	*/
10273	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10274	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10275	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10276	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10277	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10278	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10279	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10280	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10281	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10282
10283	/*
10284	* We need to flush the TLB if we are switching the APIC-access page address.
10285	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10286	*/
10287	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10288	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10289
10290	/*
10291	* MSR bitmap.
10292	*
10293	* The MSR bitmap address has already been initialized while setting up the nested-guest
10294	* VMCS, here we need to merge the MSR bitmaps.
10295	*/
10296	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10297	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10298
10299	return VINF_SUCCESS;
10300	}
10301	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10302
10303
10304	/**
10305	* Does the preparations before executing guest code in VT-x.
10306	*
10307	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10308	* recompiler/IEM. We must be cautious what we do here regarding committing
10309	* guest-state information into the VMCS assuming we assuredly execute the
10310	* guest in VT-x mode.
10311	*
10312	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10313	* the common-state (TRPM/forceflags), we must undo those changes so that the
10314	* recompiler/IEM can (and should) use them when it resumes guest execution.
10315	* Otherwise such operations must be done when we can no longer exit to ring-3.
10316	*
10317	* @returns Strict VBox status code (i.e. informational status codes too).
10318	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10319	* have been disabled.
10320	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10321	* pending events).
10322	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10323	* double-fault into the guest.
10324	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10325	* dispatched directly.
10326	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10327	*
10328	* @param pVCpu The cross context virtual CPU structure.
10329	* @param pVmxTransient The VMX-transient structure.
10330	* @param fStepping Whether we are single-stepping the guest in the
10331	* hypervisor debugger. Makes us ignore some of the reasons
10332	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10333	* if event dispatching took place.
10334	*/
10335	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10336	{
10337	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10338
10339	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10340	if (pVmxTransient->fIsNestedGuest)
10341	{
10342	RT_NOREF2(pVCpu, fStepping);
10343	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10344	return VINF_EM_RESCHEDULE_REM;
10345	}
10346	#endif
10347
10348	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10349	PGMRZDynMapFlushAutoSet(pVCpu);
10350	#endif
10351
10352	/*
10353	* Check and process force flag actions, some of which might require us to go back to ring-3.
10354	*/
10355	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10356	if (rcStrict == VINF_SUCCESS)
10357	{
10358	/* FFs don't get set all the time. */
10359	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10360	if ( pVmxTransient->fIsNestedGuest
10361	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10362	{
10363	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10364	return VINF_VMX_VMEXIT;
10365	}
10366	#endif
10367	}
10368	else
10369	return rcStrict;
10370
10371	/*
10372	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10373	*/
10374	/** @todo Doing this from ring-3 after VM setup phase causes a
10375	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
10376	* idea why atm. */
10377	PVM pVM = pVCpu->CTX_SUFF(pVM);
10378	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10379	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10380	&& PDMHasApic(pVM))
10381	{
10382	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10383	AssertRCReturn(rc, rc);
10384	}
10385
10386	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10387	/*
10388	* Merge guest VMCS controls with the nested-guest VMCS controls.
10389	*
10390	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10391	* saved state), we should be okay with merging controls as we initialize the
10392	* guest VMCS controls as part of VM setup phase.
10393	*/
10394	if ( pVmxTransient->fIsNestedGuest
10395	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10396	{
10397	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10398	AssertRCReturn(rc, rc);
10399	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10400	}
10401	#endif
10402
10403	/*
10404	* Evaluate events to be injected into the guest.
10405	*
10406	* Events in TRPM can be injected without inspecting the guest state.
10407	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10408	* guest to cause a VM-exit the next time they are ready to receive the event.
10409	*
10410	* With nested-guests, evaluating pending events may cause VM-exits.
10411	*/
10412	if (TRPMHasTrap(pVCpu))
10413	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10414
10415	uint32_t fIntrState;
10416	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10417
10418	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10419	/*
10420	* While evaluating pending events if something failed (unlikely) or if we were
10421	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10422	*/
10423	if (rcStrict != VINF_SUCCESS)
10424	return rcStrict;
10425	if ( pVmxTransient->fIsNestedGuest
10426	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10427	{
10428	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10429	return VINF_VMX_VMEXIT;
10430	}
10431	#else
10432	Assert(rcStrict == VINF_SUCCESS);
10433	#endif
10434
10435	/*
10436	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10437	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10438	* also result in triple-faulting the VM.
10439	*
10440	* With nested-guests, the above does not apply since unrestricted guest execution is a
10441	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10442	*/
10443	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10444	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10445	{ /* likely */ }
10446	else
10447	{
10448	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10449	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10450	return rcStrict;
10451	}
10452
10453	/*
10454	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10455	* import CR3 themselves. We will need to update them here, as even as late as the above
10456	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10457	* the below force flags to be set.
10458	*/
10459	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10460	{
10461	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10462	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10463	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10464	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10465	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10466	}
10467	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10468	{
10469	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10470	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10471	}
10472
10473	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10474	/* Paranoia. */
10475	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10476	#endif
10477
10478	/*
10479	* No longjmps to ring-3 from this point on!!!
10480	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10481	* This also disables flushing of the R0-logger instance (if any).
10482	*/
10483	VMMRZCallRing3Disable(pVCpu);
10484
10485	/*
10486	* Export the guest state bits.
10487	*
10488	* We cannot perform longjmps while loading the guest state because we do not preserve the
10489	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10490	* CPU migration.
10491	*
10492	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10493	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10494	*/
10495	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10496	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10497	{ /* likely */ }
10498	else
10499	{
10500	VMMRZCallRing3Enable(pVCpu);
10501	return rcStrict;
10502	}
10503
10504	/*
10505	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10506	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10507	* preemption disabled for a while. Since this is purely to aid the
10508	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10509	* disable interrupt on NT.
10510	*
10511	* We need to check for force-flags that could've possible been altered since we last
10512	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10513	* see @bugref{6398}).
10514	*
10515	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10516	* to ring-3 before executing guest code.
10517	*/
10518	pVmxTransient->fEFlags = ASMIntDisableFlags();
10519
10520	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10521	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10522	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10523	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10524	{
10525	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10526	{
10527	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10528	/*
10529	* If we are executing a nested-guest make sure that we should intercept subsequent
10530	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10531	* the VM-exit instruction emulation happy.
10532	*/
10533	if (pVmxTransient->fIsNestedGuest)
10534	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
10535	#endif
10536
10537	/*
10538	* We've injected any pending events. This is really the point of no return (to ring-3).
10539	*
10540	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10541	* returns from this function, so do -not- enable them here.
10542	*/
10543	pVCpu->hm.s.Event.fPending = false;
10544	return VINF_SUCCESS;
10545	}
10546
10547	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10548	rcStrict = VINF_EM_RAW_INTERRUPT;
10549	}
10550	else
10551	{
10552	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10553	rcStrict = VINF_EM_RAW_TO_R3;
10554	}
10555
10556	ASMSetFlags(pVmxTransient->fEFlags);
10557	VMMRZCallRing3Enable(pVCpu);
10558
10559	return rcStrict;
10560	}
10561
10562
10563	/**
10564	* Final preparations before executing guest code using hardware-assisted VMX.
10565	*
10566	* We can no longer get preempted to a different host CPU and there are no returns
10567	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10568	* failures), this function is not intended to fail sans unrecoverable hardware
10569	* errors.
10570	*
10571	* @param pVCpu The cross context virtual CPU structure.
10572	* @param pVmxTransient The VMX-transient structure.
10573	*
10574	* @remarks Called with preemption disabled.
10575	* @remarks No-long-jump zone!!!
10576	*/
10577	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10578	{
10579	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10580	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10581	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10582	Assert(!pVCpu->hm.s.Event.fPending);
10583
10584	/*
10585	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10586	*/
10587	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10588	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10589
10590	PVM pVM = pVCpu->CTX_SUFF(pVM);
10591	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10592	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10593	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10594
10595	if (!CPUMIsGuestFPUStateActive(pVCpu))
10596	{
10597	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10598	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10599	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10600	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10601	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10602	}
10603
10604	/*
10605	* Re-export the host state bits as we may've been preempted (only happens when
10606	* thread-context hooks are used or when the VM start function changes) or if
10607	* the host CR0 is modified while loading the guest FPU state above.
10608	*
10609	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10610	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10611	* see @bugref{8432}.
10612	*
10613	* This may also happen when switching to/from a nested-guest VMCS without leaving
10614	* ring-0.
10615	*/
10616	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10617	{
10618	hmR0VmxExportHostState(pVCpu);
10619	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10620	}
10621	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10622
10623	/*
10624	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10625	*/
10626	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10627	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10628	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10629
10630	/*
10631	* Store status of the shared guest/host debug state at the time of VM-entry.
10632	*/
10633	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10634	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10635
10636	/*
10637	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10638	* more than one conditional check. The post-run side of our code shall determine
10639	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10640	*/
10641	if (pVmcsInfo->pbVirtApic)
10642	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10643
10644	/*
10645	* Update the host MSRs values in the VM-exit MSR-load area.
10646	*/
10647	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10648	{
10649	if (pVmcsInfo->cExitMsrLoad > 0)
10650	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10651	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10652	}
10653
10654	/*
10655	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10656	* VMX-preemption timer based on the next virtual sync clock deadline.
10657	*/
10658	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10659	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10660	{
10661	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10662	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10663	}
10664
10665	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10666	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10667	if (!fIsRdtscIntercepted)
10668	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10669	else
10670	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10671
10672	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10673	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10674	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10675	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10676	pVmcsInfo->idHostCpu = idCurrentCpu; /* Update the CPU for which we updated host-state in this VMCS. */
10677
10678	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10679
10680	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10681	as we're about to start executing the guest . */
10682
10683	/*
10684	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10685	*
10686	* This is done this late as updating the TSC offsetting/preemption timer above
10687	* figures out if we can skip intercepting RDTSCP by calculating the number of
10688	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10689	*/
10690	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10691	&& !fIsRdtscIntercepted)
10692	{
10693	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10694
10695	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10696	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10697	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10698	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10699	AssertRC(rc);
10700	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10701	pVmxTransient->fRemoveTscAuxMsr = true;
10702	}
10703
10704	#ifdef VBOX_STRICT
10705	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10706	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10707	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10708	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10709	#endif
10710
10711	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10712	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10713	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10714	* see @bugref{9180#c54}. */
10715	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10716	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10717	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10718	#endif
10719	}
10720
10721
10722	/**
10723	* First C routine invoked after running guest code using hardware-assisted VMX.
10724	*
10725	* @param pVCpu The cross context virtual CPU structure.
10726	* @param pVmxTransient The VMX-transient structure.
10727	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10728	*
10729	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10730	*
10731	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10732	* unconditionally when it is safe to do so.
10733	*/
10734	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10735	{
10736	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10737
10738	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10739	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10740	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10741	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10742	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10743	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10744
10745	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10746	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10747	{
10748	uint64_t uGstTsc;
10749	if (!pVmxTransient->fIsNestedGuest)
10750	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10751	else
10752	{
10753	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10754	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10755	}
10756	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10757	}
10758
10759	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10760	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10761	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10762
10763	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10764	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10765	#ifdef VBOX_STRICT
10766	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10767	#endif
10768	Assert(!ASMIntAreEnabled());
10769	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10770	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10771
10772	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
10773	/*
10774	* Clean all the VMCS fields in the transient structure before reading
10775	* anything from the VMCS.
10776	*/
10777	pVmxTransient->uExitReason = 0;
10778	pVmxTransient->uExitIntErrorCode = 0;
10779	pVmxTransient->uExitQual = 0;
10780	pVmxTransient->uGuestLinearAddr = 0;
10781	pVmxTransient->uExitIntInfo = 0;
10782	pVmxTransient->cbInstr = 0;
10783	pVmxTransient->ExitInstrInfo.u = 0;
10784	pVmxTransient->uEntryIntInfo = 0;
10785	pVmxTransient->uEntryXcptErrorCode = 0;
10786	pVmxTransient->cbEntryInstr = 0;
10787	pVmxTransient->uIdtVectoringInfo = 0;
10788	pVmxTransient->uIdtVectoringErrorCode = 0;
10789	#endif
10790
10791	/*
10792	* Save the basic VM-exit reason and check if the VM-entry failed.
10793	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10794	*/
10795	uint32_t uExitReason;
10796	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10797	AssertRC(rc);
10798	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10799	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10800
10801	/*
10802	* Log the VM-exit before logging anything else as otherwise it might be a
10803	* tad confusing what happens before and after the world-switch.
10804	*/
10805	HMVMX_LOG_EXIT(pVCpu, uExitReason);
10806
10807	/*
10808	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
10809	* bitmap permissions, if it was added before VM-entry.
10810	*/
10811	if (pVmxTransient->fRemoveTscAuxMsr)
10812	{
10813	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10814	pVmxTransient->fRemoveTscAuxMsr = false;
10815	}
10816
10817	/*
10818	* Check if VMLAUNCH/VMRESUME succeeded.
10819	* If this failed, we cause a guru meditation and cease further execution.
10820	*
10821	* However, if we are executing a nested-guest we might fail if we use the
10822	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10823	*/
10824	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10825	{
10826	/*
10827	* Update the VM-exit history array here even if the VM-entry failed due to:
10828	* - Invalid guest state.
10829	* - MSR loading.
10830	* - Machine-check event.
10831	*
10832	* In any of the above cases we will still have a "valid" VM-exit reason
10833	* despite @a fVMEntryFailed being false.
10834	*
10835	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10836	*
10837	* Note! We don't have CS or RIP at this point. Will probably address that later
10838	* by amending the history entry added here.
10839	*/
10840	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10841	UINT64_MAX, uHostTsc);
10842
10843	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10844	{
10845	VMMRZCallRing3Enable(pVCpu);
10846
10847	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10848	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10849
10850	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
10851	rc = hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
10852	AssertRC(rc);
10853	#endif
10854	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10855	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10856	AssertRC(rc);
10857	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10858	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10859	AssertRC(rc);
10860	#else
10861	/*
10862	* Import the guest-interruptibility state always as we need it while evaluating
10863	* injecting events on re-entry.
10864	*
10865	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10866	* checking for real-mode while exporting the state because all bits that cause
10867	* mode changes wrt CR0 are intercepted.
10868	*/
10869	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10870	AssertRC(rc);
10871	#endif
10872
10873	/*
10874	* Sync the TPR shadow with our APIC state.
10875	*/
10876	if ( !pVmxTransient->fIsNestedGuest
10877	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10878	{
10879	Assert(pVmcsInfo->pbVirtApic);
10880	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10881	{
10882	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10883	AssertRC(rc);
10884	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10885	}
10886	}
10887
10888	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10889	return;
10890	}
10891	}
10892	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10893	else if (pVmxTransient->fIsNestedGuest)
10894	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10895	#endif
10896	else
10897	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10898
10899	VMMRZCallRing3Enable(pVCpu);
10900	}
10901
10902
10903	/**
10904	* Runs the guest code using hardware-assisted VMX the normal way.
10905	*
10906	* @returns VBox status code.
10907	* @param pVCpu The cross context virtual CPU structure.
10908	* @param pcLoops Pointer to the number of executed loops.
10909	*/
10910	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10911	{
10912	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10913	Assert(pcLoops);
10914	Assert(*pcLoops <= cMaxResumeLoops);
10915	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10916
10917	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10918	/*
10919	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
10920	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
10921	* guest VMCS while entering the VMX ring-0 session.
10922	*/
10923	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10924	{
10925	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
10926	if (RT_SUCCESS(rc))
10927	{ /* likely */ }
10928	else
10929	{
10930	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
10931	return rc;
10932	}
10933	}
10934	#endif
10935
10936	VMXTRANSIENT VmxTransient;
10937	RT_ZERO(VmxTransient);
10938	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10939
10940	/* Paranoia. */
10941	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10942
10943	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10944	for (;;)
10945	{
10946	Assert(!HMR0SuspendPending());
10947	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10948	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10949
10950	/*
10951	* Preparatory work for running nested-guest code, this may force us to
10952	* return to ring-3.
10953	*
10954	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10955	*/
10956	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10957	if (rcStrict != VINF_SUCCESS)
10958	break;
10959
10960	/* Interrupts are disabled at this point! */
10961	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10962	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10963	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10964	/* Interrupts are re-enabled at this point! */
10965
10966	/*
10967	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10968	*/
10969	if (RT_SUCCESS(rcRun))
10970	{ /* very likely */ }
10971	else
10972	{
10973	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10974	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10975	return rcRun;
10976	}
10977
10978	/*
10979	* Profile the VM-exit.
10980	*/
10981	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10983	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10984	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10985	HMVMX_START_EXIT_DISPATCH_PROF();
10986
10987	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10988
10989	/*
10990	* Handle the VM-exit.
10991	*/
10992	#ifdef HMVMX_USE_FUNCTION_TABLE
10993	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10994	#else
10995	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10996	#endif
10997	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10998	if (rcStrict == VINF_SUCCESS)
10999	{
11000	if (++(*pcLoops) <= cMaxResumeLoops)
11001	continue;
11002	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11003	rcStrict = VINF_EM_RAW_INTERRUPT;
11004	}
11005	break;
11006	}
11007
11008	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11009	return rcStrict;
11010	}
11011
11012
11013	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11014	/**
11015	* Runs the nested-guest code using hardware-assisted VMX.
11016	*
11017	* @returns VBox status code.
11018	* @param pVCpu The cross context virtual CPU structure.
11019	* @param pcLoops Pointer to the number of executed loops.
11020	*
11021	* @sa hmR0VmxRunGuestCodeNormal.
11022	*/
11023	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11024	{
11025	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11026	Assert(pcLoops);
11027	Assert(*pcLoops <= cMaxResumeLoops);
11028	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11029
11030	/*
11031	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11032	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11033	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11034	*/
11035	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11036	{
11037	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11038	if (RT_SUCCESS(rc))
11039	{ /* likely */ }
11040	else
11041	{
11042	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11043	return rc;
11044	}
11045	}
11046
11047	VMXTRANSIENT VmxTransient;
11048	RT_ZERO(VmxTransient);
11049	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11050	VmxTransient.fIsNestedGuest = true;
11051
11052	/* Paranoia. */
11053	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11054
11055	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11056	for (;;)
11057	{
11058	Assert(!HMR0SuspendPending());
11059	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11060	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11061
11062	/*
11063	* Preparatory work for running guest code, this may force us to
11064	* return to ring-3.
11065	*
11066	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11067	*/
11068	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11069	if (rcStrict != VINF_SUCCESS)
11070	break;
11071
11072	/* Interrupts are disabled at this point! */
11073	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11074	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11075	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11076	/* Interrupts are re-enabled at this point! */
11077
11078	/*
11079	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11080	*/
11081	if (RT_SUCCESS(rcRun))
11082	{ /* very likely */ }
11083	else
11084	{
11085	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11086	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11087	return rcRun;
11088	}
11089
11090	/*
11091	* Profile the VM-exit.
11092	*/
11093	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11094	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11095	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11096	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11097	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11098	HMVMX_START_EXIT_DISPATCH_PROF();
11099
11100	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11101
11102	/*
11103	* Handle the VM-exit.
11104	*/
11105	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11106	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11107	if (rcStrict == VINF_SUCCESS)
11108	{
11109	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11110	{
11111	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11112	rcStrict = VINF_VMX_VMEXIT;
11113	}
11114	else
11115	{
11116	if (++(*pcLoops) <= cMaxResumeLoops)
11117	continue;
11118	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11119	rcStrict = VINF_EM_RAW_INTERRUPT;
11120	}
11121	}
11122	else
11123	Assert(rcStrict != VINF_VMX_VMEXIT);
11124	break;
11125	}
11126
11127	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11128	return rcStrict;
11129	}
11130	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11131
11132
11133	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11134	* probes.
11135	*
11136	* The following few functions and associated structure contains the bloat
11137	* necessary for providing detailed debug events and dtrace probes as well as
11138	* reliable host side single stepping. This works on the principle of
11139	* "subclassing" the normal execution loop and workers. We replace the loop
11140	* method completely and override selected helpers to add necessary adjustments
11141	* to their core operation.
11142	*
11143	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11144	* any performance for debug and analysis features.
11145	*
11146	* @{
11147	*/
11148
11149	/**
11150	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11151	* the debug run loop.
11152	*/
11153	typedef struct VMXRUNDBGSTATE
11154	{
11155	/** The RIP we started executing at. This is for detecting that we stepped. */
11156	uint64_t uRipStart;
11157	/** The CS we started executing with. */
11158	uint16_t uCsStart;
11159
11160	/** Whether we've actually modified the 1st execution control field. */
11161	bool fModifiedProcCtls : 1;
11162	/** Whether we've actually modified the 2nd execution control field. */
11163	bool fModifiedProcCtls2 : 1;
11164	/** Whether we've actually modified the exception bitmap. */
11165	bool fModifiedXcptBitmap : 1;
11166
11167	/** We desire the modified the CR0 mask to be cleared. */
11168	bool fClearCr0Mask : 1;
11169	/** We desire the modified the CR4 mask to be cleared. */
11170	bool fClearCr4Mask : 1;
11171	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11172	uint32_t fCpe1Extra;
11173	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11174	uint32_t fCpe1Unwanted;
11175	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11176	uint32_t fCpe2Extra;
11177	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11178	uint32_t bmXcptExtra;
11179	/** The sequence number of the Dtrace provider settings the state was
11180	* configured against. */
11181	uint32_t uDtraceSettingsSeqNo;
11182	/** VM-exits to check (one bit per VM-exit). */
11183	uint32_t bmExitsToCheck[3];
11184
11185	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11186	uint32_t fProcCtlsInitial;
11187	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11188	uint32_t fProcCtls2Initial;
11189	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11190	uint32_t bmXcptInitial;
11191	} VMXRUNDBGSTATE;
11192	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11193	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11194
11195
11196	/**
11197	* Initializes the VMXRUNDBGSTATE structure.
11198	*
11199	* @param pVCpu The cross context virtual CPU structure of the
11200	* calling EMT.
11201	* @param pVmxTransient The VMX-transient structure.
11202	* @param pDbgState The debug state to initialize.
11203	*/
11204	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11205	{
11206	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11207	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11208
11209	pDbgState->fModifiedProcCtls = false;
11210	pDbgState->fModifiedProcCtls2 = false;
11211	pDbgState->fModifiedXcptBitmap = false;
11212	pDbgState->fClearCr0Mask = false;
11213	pDbgState->fClearCr4Mask = false;
11214	pDbgState->fCpe1Extra = 0;
11215	pDbgState->fCpe1Unwanted = 0;
11216	pDbgState->fCpe2Extra = 0;
11217	pDbgState->bmXcptExtra = 0;
11218	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11219	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11220	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11221	}
11222
11223
11224	/**
11225	* Updates the VMSC fields with changes requested by @a pDbgState.
11226	*
11227	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11228	* immediately before executing guest code, i.e. when interrupts are disabled.
11229	* We don't check status codes here as we cannot easily assert or return in the
11230	* latter case.
11231	*
11232	* @param pVCpu The cross context virtual CPU structure.
11233	* @param pVmxTransient The VMX-transient structure.
11234	* @param pDbgState The debug state.
11235	*/
11236	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11237	{
11238	/*
11239	* Ensure desired flags in VMCS control fields are set.
11240	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11241	*
11242	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11243	* there should be no stale data in pCtx at this point.
11244	*/
11245	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11246	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11247	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11248	{
11249	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11250	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11251	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11252	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11253	pDbgState->fModifiedProcCtls = true;
11254	}
11255
11256	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11257	{
11258	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11259	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11260	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11261	pDbgState->fModifiedProcCtls2 = true;
11262	}
11263
11264	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11265	{
11266	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11267	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11268	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11269	pDbgState->fModifiedXcptBitmap = true;
11270	}
11271
11272	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11273	{
11274	pVmcsInfo->u64Cr0Mask = 0;
11275	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, 0);
11276	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11277	}
11278
11279	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11280	{
11281	pVmcsInfo->u64Cr4Mask = 0;
11282	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, 0);
11283	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11284	}
11285
11286	NOREF(pVCpu);
11287	}
11288
11289
11290	/**
11291	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11292	* re-entry next time around.
11293	*
11294	* @returns Strict VBox status code (i.e. informational status codes too).
11295	* @param pVCpu The cross context virtual CPU structure.
11296	* @param pVmxTransient The VMX-transient structure.
11297	* @param pDbgState The debug state.
11298	* @param rcStrict The return code from executing the guest using single
11299	* stepping.
11300	*/
11301	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11302	VBOXSTRICTRC rcStrict)
11303	{
11304	/*
11305	* Restore VM-exit control settings as we may not reenter this function the
11306	* next time around.
11307	*/
11308	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11309
11310	/* We reload the initial value, trigger what we can of recalculations the
11311	next time around. From the looks of things, that's all that's required atm. */
11312	if (pDbgState->fModifiedProcCtls)
11313	{
11314	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11315	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11316	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11317	AssertRCReturn(rc2, rc2);
11318	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11319	}
11320
11321	/* We're currently the only ones messing with this one, so just restore the
11322	cached value and reload the field. */
11323	if ( pDbgState->fModifiedProcCtls2
11324	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11325	{
11326	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11327	AssertRCReturn(rc2, rc2);
11328	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11329	}
11330
11331	/* If we've modified the exception bitmap, we restore it and trigger
11332	reloading and partial recalculation the next time around. */
11333	if (pDbgState->fModifiedXcptBitmap)
11334	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11335
11336	return rcStrict;
11337	}
11338
11339
11340	/**
11341	* Configures VM-exit controls for current DBGF and DTrace settings.
11342	*
11343	* This updates @a pDbgState and the VMCS execution control fields to reflect
11344	* the necessary VM-exits demanded by DBGF and DTrace.
11345	*
11346	* @param pVCpu The cross context virtual CPU structure.
11347	* @param pVmxTransient The VMX-transient structure. May update
11348	* fUpdatedTscOffsettingAndPreemptTimer.
11349	* @param pDbgState The debug state.
11350	*/
11351	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11352	{
11353	/*
11354	* Take down the dtrace serial number so we can spot changes.
11355	*/
11356	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11357	ASMCompilerBarrier();
11358
11359	/*
11360	* We'll rebuild most of the middle block of data members (holding the
11361	* current settings) as we go along here, so start by clearing it all.
11362	*/
11363	pDbgState->bmXcptExtra = 0;
11364	pDbgState->fCpe1Extra = 0;
11365	pDbgState->fCpe1Unwanted = 0;
11366	pDbgState->fCpe2Extra = 0;
11367	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11368	pDbgState->bmExitsToCheck[i] = 0;
11369
11370	/*
11371	* Software interrupts (INT XXh) - no idea how to trigger these...
11372	*/
11373	PVM pVM = pVCpu->CTX_SUFF(pVM);
11374	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11375	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11376	{
11377	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11378	}
11379
11380	/*
11381	* INT3 breakpoints - triggered by #BP exceptions.
11382	*/
11383	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11384	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11385
11386	/*
11387	* Exception bitmap and XCPT events+probes.
11388	*/
11389	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11390	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11391	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11392
11393	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11394	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11395	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11396	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11397	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11398	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11399	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11400	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11401	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11402	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11403	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11404	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11405	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11406	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11407	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11408	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11409	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11410	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11411
11412	if (pDbgState->bmXcptExtra)
11413	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11414
11415	/*
11416	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11417	*
11418	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11419	* So, when adding/changing/removing please don't forget to update it.
11420	*
11421	* Some of the macros are picking up local variables to save horizontal space,
11422	* (being able to see it in a table is the lesser evil here).
11423	*/
11424	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11425	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11426	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11427	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11428	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11429	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11430	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11431	} else do { } while (0)
11432	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11433	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11434	{ \
11435	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11436	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11437	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11438	} else do { } while (0)
11439	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11440	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11441	{ \
11442	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11443	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11444	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11445	} else do { } while (0)
11446	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11447	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11448	{ \
11449	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11450	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11451	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11452	} else do { } while (0)
11453
11454	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11455	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11456	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11457	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11458	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11459
11460	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11461	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11462	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11463	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11464	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11465	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11466	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11467	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11468	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11469	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11470	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11471	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11472	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11473	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11474	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11475	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11476	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11477	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11478	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11479	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11480	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11481	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11482	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11483	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11484	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11485	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11486	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11487	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11488	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11489	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11490	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11491	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11492	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11493	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11494	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11495	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11496
11497	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11498	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11499	{
11500	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11501	\| CPUMCTX_EXTRN_APIC_TPR);
11502	AssertRC(rc);
11503
11504	#if 0 /** @todo fix me */
11505	pDbgState->fClearCr0Mask = true;
11506	pDbgState->fClearCr4Mask = true;
11507	#endif
11508	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11509	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11510	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11511	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11512	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11513	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11514	require clearing here and in the loop if we start using it. */
11515	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11516	}
11517	else
11518	{
11519	if (pDbgState->fClearCr0Mask)
11520	{
11521	pDbgState->fClearCr0Mask = false;
11522	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11523	}
11524	if (pDbgState->fClearCr4Mask)
11525	{
11526	pDbgState->fClearCr4Mask = false;
11527	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11528	}
11529	}
11530	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11531	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11532
11533	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11534	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11535	{
11536	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11537	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11538	}
11539	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11540	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11541
11542	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11543	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11544	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11545	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11546	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11547	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11548	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11549	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11550	#if 0 /** @todo too slow, fix handler. */
11551	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11552	#endif
11553	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11554
11555	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11556	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11557	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11558	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11559	{
11560	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11561	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11562	}
11563	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11564	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11565	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11566	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11567
11568	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11569	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11570	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11571	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11572	{
11573	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11574	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11575	}
11576	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11577	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11578	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11579	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11580
11581	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11582	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11583	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11584	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11585	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11586	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11587	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11588	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11589	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11590	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11591	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11592	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11593	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11594	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11595	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11596	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11597	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11598	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11599	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11600	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11601	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11602	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11603
11604	#undef IS_EITHER_ENABLED
11605	#undef SET_ONLY_XBM_IF_EITHER_EN
11606	#undef SET_CPE1_XBM_IF_EITHER_EN
11607	#undef SET_CPEU_XBM_IF_EITHER_EN
11608	#undef SET_CPE2_XBM_IF_EITHER_EN
11609
11610	/*
11611	* Sanitize the control stuff.
11612	*/
11613	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11614	if (pDbgState->fCpe2Extra)
11615	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11616	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11617	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11618	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11619	{
11620	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11621	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11622	}
11623
11624	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11625	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11626	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11627	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11628	}
11629
11630
11631	/**
11632	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11633	* appropriate.
11634	*
11635	* The caller has checked the VM-exit against the
11636	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11637	* already, so we don't have to do that either.
11638	*
11639	* @returns Strict VBox status code (i.e. informational status codes too).
11640	* @param pVCpu The cross context virtual CPU structure.
11641	* @param pVmxTransient The VMX-transient structure.
11642	* @param uExitReason The VM-exit reason.
11643	*
11644	* @remarks The name of this function is displayed by dtrace, so keep it short
11645	* and to the point. No longer than 33 chars long, please.
11646	*/
11647	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11648	{
11649	/*
11650	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11651	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11652	*
11653	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11654	* does. Must add/change/remove both places. Same ordering, please.
11655	*
11656	* Added/removed events must also be reflected in the next section
11657	* where we dispatch dtrace events.
11658	*/
11659	bool fDtrace1 = false;
11660	bool fDtrace2 = false;
11661	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11662	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11663	uint32_t uEventArg = 0;
11664	#define SET_EXIT(a_EventSubName) \
11665	do { \
11666	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11667	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11668	} while (0)
11669	#define SET_BOTH(a_EventSubName) \
11670	do { \
11671	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11672	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11673	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11674	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11675	} while (0)
11676	switch (uExitReason)
11677	{
11678	case VMX_EXIT_MTF:
11679	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11680
11681	case VMX_EXIT_XCPT_OR_NMI:
11682	{
11683	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11684	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11685	{
11686	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11687	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11688	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11689	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11690	{
11691	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11692	{
11693	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11694	uEventArg = pVmxTransient->uExitIntErrorCode;
11695	}
11696	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11697	switch (enmEvent1)
11698	{
11699	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11700	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11701	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11702	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11703	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11704	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11705	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11706	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11707	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11708	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11709	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11710	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11711	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11712	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11713	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11714	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11715	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11716	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11717	default: break;
11718	}
11719	}
11720	else
11721	AssertFailed();
11722	break;
11723
11724	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11725	uEventArg = idxVector;
11726	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11727	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11728	break;
11729	}
11730	break;
11731	}
11732
11733	case VMX_EXIT_TRIPLE_FAULT:
11734	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11735	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11736	break;
11737	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11738	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11739	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11740	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11741	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11742
11743	/* Instruction specific VM-exits: */
11744	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11745	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11746	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11747	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11748	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11749	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11750	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11751	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11752	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11753	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11754	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11755	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11756	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11757	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11758	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11759	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11760	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11761	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11762	case VMX_EXIT_MOV_CRX:
11763	hmR0VmxReadExitQualVmcs(pVmxTransient);
11764	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11765	SET_BOTH(CRX_READ);
11766	else
11767	SET_BOTH(CRX_WRITE);
11768	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11769	break;
11770	case VMX_EXIT_MOV_DRX:
11771	hmR0VmxReadExitQualVmcs(pVmxTransient);
11772	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11773	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11774	SET_BOTH(DRX_READ);
11775	else
11776	SET_BOTH(DRX_WRITE);
11777	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11778	break;
11779	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11780	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11781	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11782	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11783	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11784	case VMX_EXIT_GDTR_IDTR_ACCESS:
11785	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11786	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11787	{
11788	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11789	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11790	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11791	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11792	}
11793	break;
11794
11795	case VMX_EXIT_LDTR_TR_ACCESS:
11796	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11797	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11798	{
11799	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11800	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11801	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11802	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11803	}
11804	break;
11805
11806	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11807	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11808	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11809	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11810	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11811	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11812	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11813	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11814	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11815	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11816	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11817
11818	/* Events that aren't relevant at this point. */
11819	case VMX_EXIT_EXT_INT:
11820	case VMX_EXIT_INT_WINDOW:
11821	case VMX_EXIT_NMI_WINDOW:
11822	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11823	case VMX_EXIT_PREEMPT_TIMER:
11824	case VMX_EXIT_IO_INSTR:
11825	break;
11826
11827	/* Errors and unexpected events. */
11828	case VMX_EXIT_INIT_SIGNAL:
11829	case VMX_EXIT_SIPI:
11830	case VMX_EXIT_IO_SMI:
11831	case VMX_EXIT_SMI:
11832	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11833	case VMX_EXIT_ERR_MSR_LOAD:
11834	case VMX_EXIT_ERR_MACHINE_CHECK:
11835	case VMX_EXIT_PML_FULL:
11836	case VMX_EXIT_VIRTUALIZED_EOI:
11837	break;
11838
11839	default:
11840	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11841	break;
11842	}
11843	#undef SET_BOTH
11844	#undef SET_EXIT
11845
11846	/*
11847	* Dtrace tracepoints go first. We do them here at once so we don't
11848	* have to copy the guest state saving and stuff a few dozen times.
11849	* Down side is that we've got to repeat the switch, though this time
11850	* we use enmEvent since the probes are a subset of what DBGF does.
11851	*/
11852	if (fDtrace1 \|\| fDtrace2)
11853	{
11854	hmR0VmxReadExitQualVmcs(pVmxTransient);
11855	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11856	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11857	switch (enmEvent1)
11858	{
11859	/** @todo consider which extra parameters would be helpful for each probe. */
11860	case DBGFEVENT_END: break;
11861	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11862	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11863	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11864	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11865	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11866	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11867	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11868	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11869	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11870	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11871	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11872	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11873	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11874	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11875	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11876	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11877	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11878	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11879	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11880	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11881	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11882	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11883	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11884	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11885	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11886	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11887	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11888	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11889	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11890	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11891	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11892	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11893	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11894	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11895	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11896	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11897	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11898	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11899	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11900	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11901	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11902	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11903	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11904	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11905	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11906	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11907	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11908	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11909	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11910	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11911	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11912	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11913	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11914	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11915	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11916	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11917	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11918	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11919	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11920	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11921	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11922	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11923	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11924	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11925	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11926	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11927	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11928	}
11929	switch (enmEvent2)
11930	{
11931	/** @todo consider which extra parameters would be helpful for each probe. */
11932	case DBGFEVENT_END: break;
11933	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11934	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11935	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11936	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11937	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11938	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11939	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11940	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11941	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11942	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11943	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11944	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11945	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11946	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11947	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11948	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11949	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11950	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11951	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11952	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11953	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11954	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11955	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11956	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11957	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11958	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11959	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11960	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11961	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11962	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11963	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11964	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11965	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11966	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11967	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11968	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11969	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11970	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11971	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11972	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11973	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11974	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11975	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11976	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11977	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11978	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11979	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11980	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11981	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11982	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11983	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11984	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11985	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11986	}
11987	}
11988
11989	/*
11990	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11991	* the DBGF call will do a full check).
11992	*
11993	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11994	* Note! If we have to events, we prioritize the first, i.e. the instruction
11995	* one, in order to avoid event nesting.
11996	*/
11997	PVM pVM = pVCpu->CTX_SUFF(pVM);
11998	if ( enmEvent1 != DBGFEVENT_END
11999	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12000	{
12001	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12002	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12003	if (rcStrict != VINF_SUCCESS)
12004	return rcStrict;
12005	}
12006	else if ( enmEvent2 != DBGFEVENT_END
12007	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12008	{
12009	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12010	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12011	if (rcStrict != VINF_SUCCESS)
12012	return rcStrict;
12013	}
12014
12015	return VINF_SUCCESS;
12016	}
12017
12018
12019	/**
12020	* Single-stepping VM-exit filtering.
12021	*
12022	* This is preprocessing the VM-exits and deciding whether we've gotten far
12023	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12024	* handling is performed.
12025	*
12026	* @returns Strict VBox status code (i.e. informational status codes too).
12027	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12028	* @param pVmxTransient The VMX-transient structure.
12029	* @param pDbgState The debug state.
12030	*/
12031	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12032	{
12033	/*
12034	* Expensive (saves context) generic dtrace VM-exit probe.
12035	*/
12036	uint32_t const uExitReason = pVmxTransient->uExitReason;
12037	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12038	{ /* more likely */ }
12039	else
12040	{
12041	hmR0VmxReadExitQualVmcs(pVmxTransient);
12042	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12043	AssertRC(rc);
12044	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12045	}
12046
12047	/*
12048	* Check for host NMI, just to get that out of the way.
12049	*/
12050	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12051	{ /* normally likely */ }
12052	else
12053	{
12054	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12055	AssertRCReturn(rc2, rc2);
12056	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12057	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12058	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12059	}
12060
12061	/*
12062	* Check for single stepping event if we're stepping.
12063	*/
12064	if (pVCpu->hm.s.fSingleInstruction)
12065	{
12066	switch (uExitReason)
12067	{
12068	case VMX_EXIT_MTF:
12069	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12070
12071	/* Various events: */
12072	case VMX_EXIT_XCPT_OR_NMI:
12073	case VMX_EXIT_EXT_INT:
12074	case VMX_EXIT_TRIPLE_FAULT:
12075	case VMX_EXIT_INT_WINDOW:
12076	case VMX_EXIT_NMI_WINDOW:
12077	case VMX_EXIT_TASK_SWITCH:
12078	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12079	case VMX_EXIT_APIC_ACCESS:
12080	case VMX_EXIT_EPT_VIOLATION:
12081	case VMX_EXIT_EPT_MISCONFIG:
12082	case VMX_EXIT_PREEMPT_TIMER:
12083
12084	/* Instruction specific VM-exits: */
12085	case VMX_EXIT_CPUID:
12086	case VMX_EXIT_GETSEC:
12087	case VMX_EXIT_HLT:
12088	case VMX_EXIT_INVD:
12089	case VMX_EXIT_INVLPG:
12090	case VMX_EXIT_RDPMC:
12091	case VMX_EXIT_RDTSC:
12092	case VMX_EXIT_RSM:
12093	case VMX_EXIT_VMCALL:
12094	case VMX_EXIT_VMCLEAR:
12095	case VMX_EXIT_VMLAUNCH:
12096	case VMX_EXIT_VMPTRLD:
12097	case VMX_EXIT_VMPTRST:
12098	case VMX_EXIT_VMREAD:
12099	case VMX_EXIT_VMRESUME:
12100	case VMX_EXIT_VMWRITE:
12101	case VMX_EXIT_VMXOFF:
12102	case VMX_EXIT_VMXON:
12103	case VMX_EXIT_MOV_CRX:
12104	case VMX_EXIT_MOV_DRX:
12105	case VMX_EXIT_IO_INSTR:
12106	case VMX_EXIT_RDMSR:
12107	case VMX_EXIT_WRMSR:
12108	case VMX_EXIT_MWAIT:
12109	case VMX_EXIT_MONITOR:
12110	case VMX_EXIT_PAUSE:
12111	case VMX_EXIT_GDTR_IDTR_ACCESS:
12112	case VMX_EXIT_LDTR_TR_ACCESS:
12113	case VMX_EXIT_INVEPT:
12114	case VMX_EXIT_RDTSCP:
12115	case VMX_EXIT_INVVPID:
12116	case VMX_EXIT_WBINVD:
12117	case VMX_EXIT_XSETBV:
12118	case VMX_EXIT_RDRAND:
12119	case VMX_EXIT_INVPCID:
12120	case VMX_EXIT_VMFUNC:
12121	case VMX_EXIT_RDSEED:
12122	case VMX_EXIT_XSAVES:
12123	case VMX_EXIT_XRSTORS:
12124	{
12125	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12126	AssertRCReturn(rc, rc);
12127	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12128	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12129	return VINF_EM_DBG_STEPPED;
12130	break;
12131	}
12132
12133	/* Errors and unexpected events: */
12134	case VMX_EXIT_INIT_SIGNAL:
12135	case VMX_EXIT_SIPI:
12136	case VMX_EXIT_IO_SMI:
12137	case VMX_EXIT_SMI:
12138	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12139	case VMX_EXIT_ERR_MSR_LOAD:
12140	case VMX_EXIT_ERR_MACHINE_CHECK:
12141	case VMX_EXIT_PML_FULL:
12142	case VMX_EXIT_VIRTUALIZED_EOI:
12143	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12144	break;
12145
12146	default:
12147	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12148	break;
12149	}
12150	}
12151
12152	/*
12153	* Check for debugger event breakpoints and dtrace probes.
12154	*/
12155	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12156	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12157	{
12158	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12159	if (rcStrict != VINF_SUCCESS)
12160	return rcStrict;
12161	}
12162
12163	/*
12164	* Normal processing.
12165	*/
12166	#ifdef HMVMX_USE_FUNCTION_TABLE
12167	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12168	#else
12169	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12170	#endif
12171	}
12172
12173
12174	/**
12175	* Single steps guest code using hardware-assisted VMX.
12176	*
12177	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12178	* but single-stepping through the hypervisor debugger.
12179	*
12180	* @returns Strict VBox status code (i.e. informational status codes too).
12181	* @param pVCpu The cross context virtual CPU structure.
12182	* @param pcLoops Pointer to the number of executed loops.
12183	*
12184	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12185	*/
12186	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12187	{
12188	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12189	Assert(pcLoops);
12190	Assert(*pcLoops <= cMaxResumeLoops);
12191
12192	VMXTRANSIENT VmxTransient;
12193	RT_ZERO(VmxTransient);
12194	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12195
12196	/* Set HMCPU indicators. */
12197	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12198	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12199	pVCpu->hm.s.fDebugWantRdTscExit = false;
12200	pVCpu->hm.s.fUsingDebugLoop = true;
12201
12202	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12203	VMXRUNDBGSTATE DbgState;
12204	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12205	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12206
12207	/*
12208	* The loop.
12209	*/
12210	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12211	for (;;)
12212	{
12213	Assert(!HMR0SuspendPending());
12214	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12215	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12216	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12217
12218	/* Set up VM-execution controls the next two can respond to. */
12219	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12220
12221	/*
12222	* Preparatory work for running guest code, this may force us to
12223	* return to ring-3.
12224	*
12225	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12226	*/
12227	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12228	if (rcStrict != VINF_SUCCESS)
12229	break;
12230
12231	/* Interrupts are disabled at this point! */
12232	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12233
12234	/* Override any obnoxious code in the above two calls. */
12235	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12236
12237	/*
12238	* Finally execute the guest.
12239	*/
12240	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12241
12242	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12243	/* Interrupts are re-enabled at this point! */
12244
12245	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12246	if (RT_SUCCESS(rcRun))
12247	{ /* very likely */ }
12248	else
12249	{
12250	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12251	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12252	return rcRun;
12253	}
12254
12255	/* Profile the VM-exit. */
12256	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12257	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12258	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12259	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12260	HMVMX_START_EXIT_DISPATCH_PROF();
12261
12262	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12263
12264	/*
12265	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12266	*/
12267	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12268	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12269	if (rcStrict != VINF_SUCCESS)
12270	break;
12271	if (++(*pcLoops) > cMaxResumeLoops)
12272	{
12273	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12274	rcStrict = VINF_EM_RAW_INTERRUPT;
12275	break;
12276	}
12277
12278	/*
12279	* Stepping: Did the RIP change, if so, consider it a single step.
12280	* Otherwise, make sure one of the TFs gets set.
12281	*/
12282	if (fStepping)
12283	{
12284	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12285	AssertRC(rc);
12286	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12287	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12288	{
12289	rcStrict = VINF_EM_DBG_STEPPED;
12290	break;
12291	}
12292	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12293	}
12294
12295	/*
12296	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12297	*/
12298	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12299	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12300	}
12301
12302	/*
12303	* Clear the X86_EFL_TF if necessary.
12304	*/
12305	if (pVCpu->hm.s.fClearTrapFlag)
12306	{
12307	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12308	AssertRC(rc);
12309	pVCpu->hm.s.fClearTrapFlag = false;
12310	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12311	}
12312	/** @todo there seems to be issues with the resume flag when the monitor trap
12313	* flag is pending without being used. Seen early in bios init when
12314	* accessing APIC page in protected mode. */
12315
12316	/*
12317	* Restore VM-exit control settings as we may not re-enter this function the
12318	* next time around.
12319	*/
12320	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12321
12322	/* Restore HMCPU indicators. */
12323	pVCpu->hm.s.fUsingDebugLoop = false;
12324	pVCpu->hm.s.fDebugWantRdTscExit = false;
12325	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12326
12327	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12328	return rcStrict;
12329	}
12330
12331
12332	/** @} */
12333
12334
12335	/**
12336	* Checks if any expensive dtrace probes are enabled and we should go to the
12337	* debug loop.
12338	*
12339	* @returns true if we should use debug loop, false if not.
12340	*/
12341	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12342	{
12343	/* It's probably faster to OR the raw 32-bit counter variables together.
12344	Since the variables are in an array and the probes are next to one
12345	another (more or less), we have good locality. So, better read
12346	eight-nine cache lines ever time and only have one conditional, than
12347	128+ conditionals, right? */
12348	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12349	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12350	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12351	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12352	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12353	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12354	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12355	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12356	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12357	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12358	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12359	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12360	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12361	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12362	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12363	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12364	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12365	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12366	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12367	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12368	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12369	) != 0
12370	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12371	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12372	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12373	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12374	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12375	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12376	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12377	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12378	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12379	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12380	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12381	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12382	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12383	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12384	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12385	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12386	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12387	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12388	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12389	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12390	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12391	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12392	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12393	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12394	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12395	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12396	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12397	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12398	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12399	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12400	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12401	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12402	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12403	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12404	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12405	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12406	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12407	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12408	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12409	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12410	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12411	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12412	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12413	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12414	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12415	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12416	) != 0
12417	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12418	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12419	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12420	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12421	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12422	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12423	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12424	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12425	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12426	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12427	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12428	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12429	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12430	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12431	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12432	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12433	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12434	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12435	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12436	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12437	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12438	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12439	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12440	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12441	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12442	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12443	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12444	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12445	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12446	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12447	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12448	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12449	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12450	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12451	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12452	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12453	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12454	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12455	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12456	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12457	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12458	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12459	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12460	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12461	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12462	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12463	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12464	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12465	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12466	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12467	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12468	) != 0;
12469	}
12470
12471
12472	/**
12473	* Runs the guest using hardware-assisted VMX.
12474	*
12475	* @returns Strict VBox status code (i.e. informational status codes too).
12476	* @param pVCpu The cross context virtual CPU structure.
12477	*/
12478	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12479	{
12480	AssertPtr(pVCpu);
12481	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12482	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12483	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12484	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12485
12486	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12487
12488	VBOXSTRICTRC rcStrict;
12489	uint32_t cLoops = 0;
12490	for (;;)
12491	{
12492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12493	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12494	#else
12495	bool const fInNestedGuestMode = false;
12496	#endif
12497	if (!fInNestedGuestMode)
12498	{
12499	if ( !pVCpu->hm.s.fUseDebugLoop
12500	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12501	&& !DBGFIsStepping(pVCpu)
12502	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12503	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12504	else
12505	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12506	}
12507	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12508	else
12509	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12510
12511	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12512	{
12513	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12514	continue;
12515	}
12516	if (rcStrict == VINF_VMX_VMEXIT)
12517	{
12518	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12519	continue;
12520	}
12521	#endif
12522	break;
12523	}
12524
12525	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12526	switch (rcLoop)
12527	{
12528	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12529	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12530	}
12531
12532	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12533	if (RT_FAILURE(rc2))
12534	{
12535	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12536	rcStrict = rc2;
12537	}
12538	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12539	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12540	return rcStrict;
12541	}
12542
12543
12544	#ifndef HMVMX_USE_FUNCTION_TABLE
12545	/**
12546	* Handles a guest VM-exit from hardware-assisted VMX execution.
12547	*
12548	* @returns Strict VBox status code (i.e. informational status codes too).
12549	* @param pVCpu The cross context virtual CPU structure.
12550	* @param pVmxTransient The VMX-transient structure.
12551	*/
12552	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12553	{
12554	#ifdef DEBUG_ramshankar
12555	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12556	do { \
12557	if (a_fSave != 0) \
12558	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12559	VBOXSTRICTRC rcStrict = a_CallExpr; \
12560	if (a_fSave != 0) \
12561	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12562	return rcStrict; \
12563	} while (0)
12564	#else
12565	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12566	#endif
12567	uint32_t const uExitReason = pVmxTransient->uExitReason;
12568	switch (uExitReason)
12569	{
12570	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12571	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12572	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12573	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12574	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12575	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12576	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12577	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12578	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12579	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12580	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12581	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12582	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12583	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12584	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12585	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12586	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12587	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12588	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12589	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12590	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12591	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12592	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12593	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12594	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12595	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12596	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12597	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12598	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12599	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12600	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12601	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12602	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12603	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12604	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12605	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12606	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12607	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12608	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12609	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12610	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12611	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12612	#else
12613	case VMX_EXIT_VMCLEAR:
12614	case VMX_EXIT_VMLAUNCH:
12615	case VMX_EXIT_VMPTRLD:
12616	case VMX_EXIT_VMPTRST:
12617	case VMX_EXIT_VMREAD:
12618	case VMX_EXIT_VMRESUME:
12619	case VMX_EXIT_VMWRITE:
12620	case VMX_EXIT_VMXOFF:
12621	case VMX_EXIT_VMXON:
12622	case VMX_EXIT_INVVPID:
12623	case VMX_EXIT_INVEPT:
12624	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12625	#endif
12626
12627	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12628	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12629	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12630
12631	case VMX_EXIT_INIT_SIGNAL:
12632	case VMX_EXIT_SIPI:
12633	case VMX_EXIT_IO_SMI:
12634	case VMX_EXIT_SMI:
12635	case VMX_EXIT_ERR_MSR_LOAD:
12636	case VMX_EXIT_ERR_MACHINE_CHECK:
12637	case VMX_EXIT_PML_FULL:
12638	case VMX_EXIT_VIRTUALIZED_EOI:
12639	case VMX_EXIT_GDTR_IDTR_ACCESS:
12640	case VMX_EXIT_LDTR_TR_ACCESS:
12641	case VMX_EXIT_APIC_WRITE:
12642	case VMX_EXIT_RDRAND:
12643	case VMX_EXIT_RSM:
12644	case VMX_EXIT_VMFUNC:
12645	case VMX_EXIT_ENCLS:
12646	case VMX_EXIT_RDSEED:
12647	case VMX_EXIT_XSAVES:
12648	case VMX_EXIT_XRSTORS:
12649	case VMX_EXIT_UMWAIT:
12650	case VMX_EXIT_TPAUSE:
12651	default:
12652	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12653	}
12654	#undef VMEXIT_CALL_RET
12655	}
12656	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12657
12658
12659	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12660	/**
12661	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12662	*
12663	* @returns Strict VBox status code (i.e. informational status codes too).
12664	* @param pVCpu The cross context virtual CPU structure.
12665	* @param pVmxTransient The VMX-transient structure.
12666	*/
12667	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12668	{
12669	uint32_t const uExitReason = pVmxTransient->uExitReason;
12670	switch (uExitReason)
12671	{
12672	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12673	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12674	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12675	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12676	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12677
12678	/*
12679	* We shouldn't direct host physical interrupts to the nested-guest.
12680	*/
12681	case VMX_EXIT_EXT_INT:
12682	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12683
12684	/*
12685	* Instructions that cause VM-exits unconditionally or the condition is
12686	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12687	* happens, it's guaranteed to be a nested-guest VM-exit).
12688	*
12689	* - Provides VM-exit instruction length ONLY.
12690	*/
12691	case VMX_EXIT_CPUID: /* Unconditional. */
12692	case VMX_EXIT_VMCALL:
12693	case VMX_EXIT_GETSEC:
12694	case VMX_EXIT_INVD:
12695	case VMX_EXIT_XSETBV:
12696	case VMX_EXIT_VMLAUNCH:
12697	case VMX_EXIT_VMRESUME:
12698	case VMX_EXIT_VMXOFF:
12699	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
12700	case VMX_EXIT_VMFUNC:
12701	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12702
12703	/*
12704	* Instructions that cause VM-exits unconditionally or the condition is
12705	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12706	* happens, it's guaranteed to be a nested-guest VM-exit).
12707	*
12708	* - Provides VM-exit instruction length.
12709	* - Provides VM-exit information.
12710	* - Optionally provides Exit qualification.
12711	*
12712	* Since Exit qualification is 0 for all VM-exits where it is not
12713	* applicable, reading and passing it to the guest should produce
12714	* defined behavior.
12715	*
12716	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12717	*/
12718	case VMX_EXIT_INVEPT: /* Unconditional. */
12719	case VMX_EXIT_INVVPID:
12720	case VMX_EXIT_VMCLEAR:
12721	case VMX_EXIT_VMPTRLD:
12722	case VMX_EXIT_VMPTRST:
12723	case VMX_EXIT_VMXON:
12724	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
12725	case VMX_EXIT_LDTR_TR_ACCESS:
12726	case VMX_EXIT_RDRAND:
12727	case VMX_EXIT_RDSEED:
12728	case VMX_EXIT_XSAVES:
12729	case VMX_EXIT_XRSTORS:
12730	case VMX_EXIT_UMWAIT:
12731	case VMX_EXIT_TPAUSE:
12732	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12733
12734	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12735	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12736	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12737	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12738	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12739	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12740	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12741	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12742	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12743	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12744	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12745	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12746	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12747	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12748	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12749	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12750	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12751	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12752	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12753
12754	case VMX_EXIT_PREEMPT_TIMER:
12755	{
12756	/** @todo NSTVMX: Preempt timer. */
12757	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12758	}
12759
12760	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12761	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12762
12763	case VMX_EXIT_VMREAD:
12764	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12765
12766	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12767	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12768
12769	case VMX_EXIT_INIT_SIGNAL:
12770	case VMX_EXIT_SIPI:
12771	case VMX_EXIT_IO_SMI:
12772	case VMX_EXIT_SMI:
12773	case VMX_EXIT_ERR_MSR_LOAD:
12774	case VMX_EXIT_ERR_MACHINE_CHECK:
12775	case VMX_EXIT_PML_FULL:
12776	case VMX_EXIT_RSM:
12777	default:
12778	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12779	}
12780	}
12781	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12782
12783
12784	/** @name VM-exit helpers.
12785	* @{
12786	*/
12787	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12788	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12789	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12790
12791	/** Macro for VM-exits called unexpectedly. */
12792	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
12793	do { \
12794	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
12795	return VERR_VMX_UNEXPECTED_EXIT; \
12796	} while (0)
12797
12798	#ifdef VBOX_STRICT
12799	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12800	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12801	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12802
12803	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12804	do { \
12805	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12806	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12807	} while (0)
12808
12809	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12810	do { \
12811	AssertPtr((a_pVCpu)); \
12812	AssertPtr((a_pVmxTransient)); \
12813	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12814	Assert((a_pVmxTransient)->pVmcsInfo); \
12815	Assert(ASMIntAreEnabled()); \
12816	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12817	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12818	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
12819	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12820	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12821	HMVMX_ASSERT_PREEMPT_CPUID(); \
12822	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12823	} while (0)
12824
12825	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12826	do { \
12827	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12828	Assert((a_pVmxTransient)->fIsNestedGuest); \
12829	} while (0)
12830
12831	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12832	do { \
12833	Log4Func(("\n")); \
12834	} while (0)
12835	#else
12836	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12837	do { \
12838	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12839	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12840	} while (0)
12841
12842	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12843	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12844
12845	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12846	#endif
12847
12848	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12849	/** Macro that does the necessary privilege checks and intercepted VM-exits for
12850	* guests that attempted to execute a VMX instruction. */
12851	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
12852	do \
12853	{ \
12854	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
12855	if (rcStrictTmp == VINF_SUCCESS) \
12856	{ /* likely */ } \
12857	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12858	{ \
12859	Assert((a_pVCpu)->hm.s.Event.fPending); \
12860	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
12861	return VINF_SUCCESS; \
12862	} \
12863	else \
12864	{ \
12865	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
12866	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
12867	} \
12868	} while (0)
12869
12870	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
12871	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
12872	do \
12873	{ \
12874	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
12875	(a_pGCPtrEffAddr)); \
12876	if (rcStrictTmp == VINF_SUCCESS) \
12877	{ /* likely */ } \
12878	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12879	{ \
12880	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
12881	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
12882	NOREF(uXcptTmp); \
12883	return VINF_SUCCESS; \
12884	} \
12885	else \
12886	{ \
12887	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
12888	return rcStrictTmp; \
12889	} \
12890	} while (0)
12891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12892
12893
12894	/**
12895	* Advances the guest RIP by the specified number of bytes.
12896	*
12897	* @param pVCpu The cross context virtual CPU structure.
12898	* @param cbInstr Number of bytes to advance the RIP by.
12899	*
12900	* @remarks No-long-jump zone!!!
12901	*/
12902	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12903	{
12904	/* Advance the RIP. */
12905	pVCpu->cpum.GstCtx.rip += cbInstr;
12906	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12907
12908	/* Update interrupt inhibition. */
12909	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12910	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12911	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12912	}
12913
12914
12915	/**
12916	* Advances the guest RIP after reading it from the VMCS.
12917	*
12918	* @returns VBox status code, no informational status codes.
12919	* @param pVCpu The cross context virtual CPU structure.
12920	* @param pVmxTransient The VMX-transient structure.
12921	*
12922	* @remarks No-long-jump zone!!!
12923	*/
12924	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12925	{
12926	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12927	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12928	AssertRCReturn(rc, rc);
12929
12930	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12931	return VINF_SUCCESS;
12932	}
12933
12934
12935	/**
12936	* Handle a condition that occurred while delivering an event through the guest or
12937	* nested-guest IDT.
12938	*
12939	* @returns Strict VBox status code (i.e. informational status codes too).
12940	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12941	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12942	* to continue execution of the guest which will delivery the \#DF.
12943	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12944	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12945	*
12946	* @param pVCpu The cross context virtual CPU structure.
12947	* @param pVmxTransient The VMX-transient structure.
12948	*
12949	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
12950	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
12951	* is due to an EPT violation, PML full or SPP-related event.
12952	*
12953	* @remarks No-long-jump zone!!!
12954	*/
12955	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12956	{
12957	Assert(!pVCpu->hm.s.Event.fPending);
12958	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
12959	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
12960	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
12961	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
12962	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
12963
12964	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12965	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12966	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
12967	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12968	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
12969	{
12970	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
12971	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
12972
12973	/*
12974	* If the event was a software interrupt (generated with INT n) or a software exception
12975	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12976	* can handle the VM-exit and continue guest execution which will re-execute the
12977	* instruction rather than re-injecting the exception, as that can cause premature
12978	* trips to ring-3 before injection and involve TRPM which currently has no way of
12979	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12980	* the problem).
12981	*/
12982	IEMXCPTRAISE enmRaise;
12983	IEMXCPTRAISEINFO fRaiseInfo;
12984	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12985	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12986	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12987	{
12988	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12989	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12990	}
12991	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
12992	{
12993	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
12994	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12995	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
12996
12997	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12998	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12999
13000	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13001
13002	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13003	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13004	{
13005	pVmxTransient->fVectoringPF = true;
13006	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13007	}
13008	}
13009	else
13010	{
13011	/*
13012	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13013	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13014	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13015	*/
13016	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13017	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13018	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13019	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13020	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13021	}
13022
13023	/*
13024	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13025	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13026	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13027	* subsequent VM-entry would fail, see @bugref{7445}.
13028	*
13029	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13030	*/
13031	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13032	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
13033	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13034	&& CPUMIsGuestNmiBlocking(pVCpu))
13035	{
13036	CPUMSetGuestNmiBlocking(pVCpu, false);
13037	}
13038
13039	switch (enmRaise)
13040	{
13041	case IEMXCPTRAISE_CURRENT_XCPT:
13042	{
13043	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
13044	Assert(rcStrict == VINF_SUCCESS);
13045	break;
13046	}
13047
13048	case IEMXCPTRAISE_PREV_EVENT:
13049	{
13050	uint32_t u32ErrCode;
13051	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13052	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13053	else
13054	u32ErrCode = 0;
13055
13056	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13057	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13058	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13059	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13060
13061	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13062	pVCpu->hm.s.Event.u32ErrCode));
13063	Assert(rcStrict == VINF_SUCCESS);
13064	break;
13065	}
13066
13067	case IEMXCPTRAISE_REEXEC_INSTR:
13068	Assert(rcStrict == VINF_SUCCESS);
13069	break;
13070
13071	case IEMXCPTRAISE_DOUBLE_FAULT:
13072	{
13073	/*
13074	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13075	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13076	*/
13077	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13078	{
13079	pVmxTransient->fVectoringDoublePF = true;
13080	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13081	pVCpu->cpum.GstCtx.cr2));
13082	rcStrict = VINF_SUCCESS;
13083	}
13084	else
13085	{
13086	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13087	hmR0VmxSetPendingXcptDF(pVCpu);
13088	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13089	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13090	rcStrict = VINF_HM_DOUBLE_FAULT;
13091	}
13092	break;
13093	}
13094
13095	case IEMXCPTRAISE_TRIPLE_FAULT:
13096	{
13097	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13098	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13099	rcStrict = VINF_EM_RESET;
13100	break;
13101	}
13102
13103	case IEMXCPTRAISE_CPU_HANG:
13104	{
13105	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13106	rcStrict = VERR_EM_GUEST_CPU_HANG;
13107	break;
13108	}
13109
13110	default:
13111	{
13112	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13113	rcStrict = VERR_VMX_IPE_2;
13114	break;
13115	}
13116	}
13117	}
13118	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13119	&& !CPUMIsGuestNmiBlocking(pVCpu))
13120	{
13121	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13122	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13123	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13124	{
13125	/*
13126	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13127	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13128	* that NMIs remain blocked until the IRET execution is completed.
13129	*
13130	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13131	*/
13132	CPUMSetGuestNmiBlocking(pVCpu, true);
13133	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13134	}
13135	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13136	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13137	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13138	{
13139	/*
13140	* Execution of IRET caused an EPT violation, page-modification log-full event or
13141	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13142	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13143	* that NMIs remain blocked until the IRET execution is completed.
13144	*
13145	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13146	*/
13147	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13148	{
13149	CPUMSetGuestNmiBlocking(pVCpu, true);
13150	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13151	}
13152	}
13153	}
13154
13155	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13156	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13157	return rcStrict;
13158	}
13159
13160
13161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13162	/**
13163	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13164	* guest attempting to execute a VMX instruction.
13165	*
13166	* @returns Strict VBox status code (i.e. informational status codes too).
13167	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13168	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13169	*
13170	* @param pVCpu The cross context virtual CPU structure.
13171	* @param uExitReason The VM-exit reason.
13172	*
13173	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13174	* @remarks No-long-jump zone!!!
13175	*/
13176	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
13177	{
13178	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13179	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13180
13181	/*
13182	* The physical CPU would have already checked the CPU mode/code segment.
13183	* We shall just assert here for paranoia.
13184	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13185	*/
13186	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13187	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13188	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13189
13190	if (uExitReason == VMX_EXIT_VMXON)
13191	{
13192	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13193
13194	/*
13195	* We check CR4.VMXE because it is required to be always set while in VMX operation
13196	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13197	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13198	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13199	*/
13200	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13201	{
13202	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13203	hmR0VmxSetPendingXcptUD(pVCpu);
13204	return VINF_HM_PENDING_XCPT;
13205	}
13206	}
13207	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13208	{
13209	/*
13210	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13211	* (other than VMXON), we need to raise a #UD.
13212	*/
13213	Log4Func(("Not in VMX root mode -> #UD\n"));
13214	hmR0VmxSetPendingXcptUD(pVCpu);
13215	return VINF_HM_PENDING_XCPT;
13216	}
13217
13218	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13219	return VINF_SUCCESS;
13220	}
13221
13222	/**
13223	* Decodes the memory operand of an instruction that caused a VM-exit.
13224	*
13225	* The Exit qualification field provides the displacement field for memory
13226	* operand instructions, if any.
13227	*
13228	* @returns Strict VBox status code (i.e. informational status codes too).
13229	* @retval VINF_SUCCESS if the operand was successfully decoded.
13230	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13231	* operand.
13232	* @param pVCpu The cross context virtual CPU structure.
13233	* @param uExitInstrInfo The VM-exit instruction information field.
13234	* @param enmMemAccess The memory operand's access type (read or write).
13235	* @param GCPtrDisp The instruction displacement field, if any. For
13236	* RIP-relative addressing pass RIP + displacement here.
13237	* @param pGCPtrMem Where to store the effective destination memory address.
13238	*
13239	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13240	* virtual-8086 mode hence skips those checks while verifying if the
13241	* segment is valid.
13242	*/
13243	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13244	PRTGCPTR pGCPtrMem)
13245	{
13246	Assert(pGCPtrMem);
13247	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13248	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13249	\| CPUMCTX_EXTRN_CR0);
13250
13251	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13252	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13253	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13254
13255	VMXEXITINSTRINFO ExitInstrInfo;
13256	ExitInstrInfo.u = uExitInstrInfo;
13257	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13258	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13259	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13260	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13261	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13262	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13263	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13264	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13265	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13266
13267	/*
13268	* Validate instruction information.
13269	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13270	*/
13271	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13272	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13273	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13274	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13275	AssertLogRelMsgReturn(fIsMemOperand,
13276	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13277
13278	/*
13279	* Compute the complete effective address.
13280	*
13281	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13282	* See AMD spec. 4.5.2 "Segment Registers".
13283	*/
13284	RTGCPTR GCPtrMem = GCPtrDisp;
13285	if (fBaseRegValid)
13286	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13287	if (fIdxRegValid)
13288	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13289
13290	RTGCPTR const GCPtrOff = GCPtrMem;
13291	if ( !fIsLongMode
13292	\|\| iSegReg >= X86_SREG_FS)
13293	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13294	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13295
13296	/*
13297	* Validate effective address.
13298	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13299	*/
13300	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13301	Assert(cbAccess > 0);
13302	if (fIsLongMode)
13303	{
13304	if (X86_IS_CANONICAL(GCPtrMem))
13305	{
13306	*pGCPtrMem = GCPtrMem;
13307	return VINF_SUCCESS;
13308	}
13309
13310	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13311	* "Data Limit Checks in 64-bit Mode". */
13312	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13313	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13314	return VINF_HM_PENDING_XCPT;
13315	}
13316
13317	/*
13318	* This is a watered down version of iemMemApplySegment().
13319	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13320	* and segment CPL/DPL checks are skipped.
13321	*/
13322	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13323	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13324	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13325
13326	/* Check if the segment is present and usable. */
13327	if ( pSel->Attr.n.u1Present
13328	&& !pSel->Attr.n.u1Unusable)
13329	{
13330	Assert(pSel->Attr.n.u1DescType);
13331	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13332	{
13333	/* Check permissions for the data segment. */
13334	if ( enmMemAccess == VMXMEMACCESS_WRITE
13335	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13336	{
13337	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13338	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13339	return VINF_HM_PENDING_XCPT;
13340	}
13341
13342	/* Check limits if it's a normal data segment. */
13343	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13344	{
13345	if ( GCPtrFirst32 > pSel->u32Limit
13346	\|\| GCPtrLast32 > pSel->u32Limit)
13347	{
13348	Log4Func(("Data segment limit exceeded. "
13349	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13350	GCPtrLast32, pSel->u32Limit));
13351	if (iSegReg == X86_SREG_SS)
13352	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13353	else
13354	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13355	return VINF_HM_PENDING_XCPT;
13356	}
13357	}
13358	else
13359	{
13360	/* Check limits if it's an expand-down data segment.
13361	Note! The upper boundary is defined by the B bit, not the G bit! */
13362	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13363	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13364	{
13365	Log4Func(("Expand-down data segment limit exceeded. "
13366	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13367	GCPtrLast32, pSel->u32Limit));
13368	if (iSegReg == X86_SREG_SS)
13369	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13370	else
13371	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13372	return VINF_HM_PENDING_XCPT;
13373	}
13374	}
13375	}
13376	else
13377	{
13378	/* Check permissions for the code segment. */
13379	if ( enmMemAccess == VMXMEMACCESS_WRITE
13380	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13381	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13382	{
13383	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13384	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13385	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13386	return VINF_HM_PENDING_XCPT;
13387	}
13388
13389	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13390	if ( GCPtrFirst32 > pSel->u32Limit
13391	\|\| GCPtrLast32 > pSel->u32Limit)
13392	{
13393	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13394	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13395	if (iSegReg == X86_SREG_SS)
13396	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13397	else
13398	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13399	return VINF_HM_PENDING_XCPT;
13400	}
13401	}
13402	}
13403	else
13404	{
13405	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13406	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13407	return VINF_HM_PENDING_XCPT;
13408	}
13409
13410	*pGCPtrMem = GCPtrMem;
13411	return VINF_SUCCESS;
13412	}
13413	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13414
13415
13416	/**
13417	* VM-exit helper for LMSW.
13418	*/
13419	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13420	{
13421	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13422	AssertRCReturn(rc, rc);
13423
13424	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13425	AssertMsg( rcStrict == VINF_SUCCESS
13426	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13427
13428	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13429	if (rcStrict == VINF_IEM_RAISED_XCPT)
13430	{
13431	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13432	rcStrict = VINF_SUCCESS;
13433	}
13434
13435	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13436	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13437	return rcStrict;
13438	}
13439
13440
13441	/**
13442	* VM-exit helper for CLTS.
13443	*/
13444	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13445	{
13446	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13447	AssertRCReturn(rc, rc);
13448
13449	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13450	AssertMsg( rcStrict == VINF_SUCCESS
13451	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13452
13453	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13454	if (rcStrict == VINF_IEM_RAISED_XCPT)
13455	{
13456	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13457	rcStrict = VINF_SUCCESS;
13458	}
13459
13460	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13461	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13462	return rcStrict;
13463	}
13464
13465
13466	/**
13467	* VM-exit helper for MOV from CRx (CRx read).
13468	*/
13469	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13470	{
13471	Assert(iCrReg < 16);
13472	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13473
13474	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13475	AssertRCReturn(rc, rc);
13476
13477	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13478	AssertMsg( rcStrict == VINF_SUCCESS
13479	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13480
13481	if (iGReg == X86_GREG_xSP)
13482	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13483	else
13484	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13485	#ifdef VBOX_WITH_STATISTICS
13486	switch (iCrReg)
13487	{
13488	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13489	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13490	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13491	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13492	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13493	}
13494	#endif
13495	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13496	return rcStrict;
13497	}
13498
13499
13500	/**
13501	* VM-exit helper for MOV to CRx (CRx write).
13502	*/
13503	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13504	{
13505	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13506	AssertRCReturn(rc, rc);
13507
13508	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13509	AssertMsg( rcStrict == VINF_SUCCESS
13510	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13511	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13512
13513	switch (iCrReg)
13514	{
13515	case 0:
13516	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13517	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13518	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13519	break;
13520
13521	case 2:
13522	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13523	/* Nothing to do here, CR2 it's not part of the VMCS. */
13524	break;
13525
13526	case 3:
13527	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13529	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13530	break;
13531
13532	case 4:
13533	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13534	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13535	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13536	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13537	break;
13538
13539	case 8:
13540	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13541	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13542	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13543	break;
13544
13545	default:
13546	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13547	break;
13548	}
13549
13550	if (rcStrict == VINF_IEM_RAISED_XCPT)
13551	{
13552	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13553	rcStrict = VINF_SUCCESS;
13554	}
13555	return rcStrict;
13556	}
13557
13558
13559	/**
13560	* VM-exit exception handler for \#PF (Page-fault exception).
13561	*
13562	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13563	*/
13564	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13565	{
13566	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13567	PVM pVM = pVCpu->CTX_SUFF(pVM);
13568	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
13569	AssertRCReturn(rc, rc);
13570
13571	if (!pVM->hm.s.fNestedPaging)
13572	{ /* likely */ }
13573	else
13574	{
13575	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13576	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13577	#endif
13578	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13579	if (!pVmxTransient->fVectoringDoublePF)
13580	{
13581	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13582	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13583	}
13584	else
13585	{
13586	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13587	Assert(!pVmxTransient->fIsNestedGuest);
13588	hmR0VmxSetPendingXcptDF(pVCpu);
13589	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13590	}
13591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13592	return rc;
13593	}
13594
13595	Assert(!pVmxTransient->fIsNestedGuest);
13596
13597	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13598	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13599	if (pVmxTransient->fVectoringPF)
13600	{
13601	Assert(pVCpu->hm.s.Event.fPending);
13602	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13603	}
13604
13605	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13606	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13607	AssertRCReturn(rc, rc);
13608
13609	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13610	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13611
13612	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13613	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13614
13615	Log4Func(("#PF: rc=%Rrc\n", rc));
13616	if (rc == VINF_SUCCESS)
13617	{
13618	/*
13619	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13620	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13621	*/
13622	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13623	TRPMResetTrap(pVCpu);
13624	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13625	return rc;
13626	}
13627
13628	if (rc == VINF_EM_RAW_GUEST_TRAP)
13629	{
13630	if (!pVmxTransient->fVectoringDoublePF)
13631	{
13632	/* It's a guest page fault and needs to be reflected to the guest. */
13633	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13634	TRPMResetTrap(pVCpu);
13635	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13636	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13637	uGstErrorCode, pVmxTransient->uExitQual);
13638	}
13639	else
13640	{
13641	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13642	TRPMResetTrap(pVCpu);
13643	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13644	hmR0VmxSetPendingXcptDF(pVCpu);
13645	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13646	}
13647
13648	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13649	return VINF_SUCCESS;
13650	}
13651
13652	TRPMResetTrap(pVCpu);
13653	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13654	return rc;
13655	}
13656
13657
13658	/**
13659	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13660	*
13661	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13662	*/
13663	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13664	{
13665	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13667
13668	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13669	AssertRCReturn(rc, rc);
13670
13671	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13672	{
13673	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13674	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13675
13676	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13677	* provides VM-exit instruction length. If this causes problem later,
13678	* disassemble the instruction like it's done on AMD-V. */
13679	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13680	AssertRCReturn(rc2, rc2);
13681	return rc;
13682	}
13683
13684	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13685	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13686	return VINF_SUCCESS;
13687	}
13688
13689
13690	/**
13691	* VM-exit exception handler for \#BP (Breakpoint exception).
13692	*
13693	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13694	*/
13695	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13696	{
13697	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13698	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13699
13700	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13701	AssertRCReturn(rc, rc);
13702
13703	if (!pVmxTransient->fIsNestedGuest)
13704	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13705	else
13706	rc = VINF_EM_RAW_GUEST_TRAP;
13707	if (rc == VINF_EM_RAW_GUEST_TRAP)
13708	{
13709	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13710	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13711	}
13712
13713	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13714	return rc;
13715	}
13716
13717
13718	/**
13719	* VM-exit exception handler for \#AC (Alignment-check exception).
13720	*
13721	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13722	*/
13723	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13724	{
13725	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13726	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13727
13728	/* Re-inject it. We'll detect any nesting before getting here. */
13729	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13730	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13731	return VINF_SUCCESS;
13732	}
13733
13734
13735	/**
13736	* VM-exit exception handler for \#DB (Debug exception).
13737	*
13738	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13739	*/
13740	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13741	{
13742	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13743	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13744
13745	/*
13746	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13747	*/
13748	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
13749	AssertRCReturn(rc, rc);
13750
13751	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13752	uint64_t const uDR6 = X86_DR6_INIT_VAL
13753	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13754	\| X86_DR6_BD \| X86_DR6_BS));
13755
13756	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13757	if (!pVmxTransient->fIsNestedGuest)
13758	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13759	else
13760	rc = VINF_EM_RAW_GUEST_TRAP;
13761	Log6Func(("rc=%Rrc\n", rc));
13762	if (rc == VINF_EM_RAW_GUEST_TRAP)
13763	{
13764	/*
13765	* The exception was for the guest. Update DR6, DR7.GD and
13766	* IA32_DEBUGCTL.LBR before forwarding it.
13767	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
13768	*/
13769	VMMRZCallRing3Disable(pVCpu);
13770	HM_DISABLE_PREEMPT(pVCpu);
13771
13772	pCtx->dr[6] &= ~X86_DR6_B_MASK;
13773	pCtx->dr[6] \|= uDR6;
13774	if (CPUMIsGuestDebugStateActive(pVCpu))
13775	ASMSetDR6(pCtx->dr[6]);
13776
13777	HM_RESTORE_PREEMPT();
13778	VMMRZCallRing3Enable(pVCpu);
13779
13780	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
13781	AssertRCReturn(rc, rc);
13782
13783	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13784	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
13785
13786	/* Paranoia. */
13787	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
13788	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
13789
13790	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
13791	AssertRCReturn(rc, rc);
13792
13793	/*
13794	* Raise #DB in the guest.
13795	*
13796	* It is important to reflect exactly what the VM-exit gave us (preserving the
13797	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13798	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13799	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13800	*
13801	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13802	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13803	*/
13804	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13805	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13806	return VINF_SUCCESS;
13807	}
13808
13809	/*
13810	* Not a guest trap, must be a hypervisor related debug event then.
13811	* Update DR6 in case someone is interested in it.
13812	*/
13813	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13814	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13815	CPUMSetHyperDR6(pVCpu, uDR6);
13816
13817	return rc;
13818	}
13819
13820
13821	/**
13822	* Hacks its way around the lovely mesa driver's backdoor accesses.
13823	*
13824	* @sa hmR0SvmHandleMesaDrvGp.
13825	*/
13826	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13827	{
13828	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
13829	RT_NOREF(pCtx);
13830
13831	/* For now we'll just skip the instruction. */
13832	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13833	}
13834
13835
13836	/**
13837	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
13838	* backdoor logging w/o checking what it is running inside.
13839	*
13840	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
13841	* backdoor port and magic numbers loaded in registers.
13842	*
13843	* @returns true if it is, false if it isn't.
13844	* @sa hmR0SvmIsMesaDrvGp.
13845	*/
13846	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13847	{
13848	/* 0xed: IN eAX,dx */
13849	uint8_t abInstr[1];
13850	if (pVmxTransient->cbInstr != sizeof(abInstr))
13851	return false;
13852
13853	/* Check that it is #GP(0). */
13854	if (pVmxTransient->uExitIntErrorCode != 0)
13855	return false;
13856
13857	/* Check magic and port. */
13858	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
13859	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
13860	if (pCtx->rax != UINT32_C(0x564d5868))
13861	return false;
13862	if (pCtx->dx != UINT32_C(0x5658))
13863	return false;
13864
13865	/* Flat ring-3 CS. */
13866	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
13867	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
13868	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
13869	if (pCtx->cs.Attr.n.u2Dpl != 3)
13870	return false;
13871	if (pCtx->cs.u64Base != 0)
13872	return false;
13873
13874	/* Check opcode. */
13875	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
13876	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
13877	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
13878	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
13879	if (RT_FAILURE(rc))
13880	return false;
13881	if (abInstr[0] != 0xed)
13882	return false;
13883
13884	return true;
13885	}
13886
13887
13888	/**
13889	* VM-exit exception handler for \#GP (General-protection exception).
13890	*
13891	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13892	*/
13893	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13894	{
13895	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13896	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13897
13898	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13899	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13900	if (pVmcsInfo->RealMode.fRealOnV86Active)
13901	{ /* likely */ }
13902	else
13903	{
13904	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13905	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
13906	#endif
13907	/*
13908	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
13909	* executing a nested-guest, reflect #GP to the guest or nested-guest.
13910	*/
13911	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13912	AssertRCReturn(rc, rc);
13913	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13914	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13915
13916	if ( pVmxTransient->fIsNestedGuest
13917	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
13918	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
13919	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13920	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13921	else
13922	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
13923	return rc;
13924	}
13925
13926	Assert(CPUMIsGuestInRealModeEx(pCtx));
13927	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13928	Assert(!pVmxTransient->fIsNestedGuest);
13929
13930	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13931	AssertRCReturn(rc, rc);
13932
13933	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13934	if (rcStrict == VINF_SUCCESS)
13935	{
13936	if (!CPUMIsGuestInRealModeEx(pCtx))
13937	{
13938	/*
13939	* The guest is no longer in real-mode, check if we can continue executing the
13940	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13941	*/
13942	pVmcsInfo->RealMode.fRealOnV86Active = false;
13943	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
13944	{
13945	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
13946	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13947	}
13948	else
13949	{
13950	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13951	rcStrict = VINF_EM_RESCHEDULE;
13952	}
13953	}
13954	else
13955	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13956	}
13957	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13958	{
13959	rcStrict = VINF_SUCCESS;
13960	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13961	}
13962	return VBOXSTRICTRC_VAL(rcStrict);
13963	}
13964
13965
13966	/**
13967	* VM-exit exception handler wrapper for all other exceptions that are not handled
13968	* by a specific handler.
13969	*
13970	* This simply re-injects the exception back into the VM without any special
13971	* processing.
13972	*
13973	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13974	*/
13975	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13976	{
13977	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13978
13979	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13980	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13981	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
13982	("uVector=%#x u32XcptBitmap=%#X32\n",
13983	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
13984	NOREF(pVmcsInfo);
13985	#endif
13986
13987	/*
13988	* Re-inject the exception into the guest. This cannot be a double-fault condition which
13989	* would have been handled while checking exits due to event delivery.
13990	*/
13991	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13992
13993	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13994	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13995	AssertRCReturn(rc, rc);
13996	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13997	#endif
13998
13999	#ifdef VBOX_WITH_STATISTICS
14000	switch (uVector)
14001	{
14002	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14003	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14004	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14005	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14006	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14007	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14008	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14009	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14010	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14011	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14012	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14013	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14014	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14015	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14016	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14017	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14018	default:
14019	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14020	break;
14021	}
14022	#endif
14023
14024	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14025	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
14026	NOREF(uVector);
14027
14028	/* Re-inject the original exception into the guest. */
14029	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14030	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14031	return VINF_SUCCESS;
14032	}
14033
14034
14035	/**
14036	* VM-exit exception handler for all exceptions (except NMIs!).
14037	*
14038	* @remarks This may be called for both guests and nested-guests. Take care to not
14039	* make assumptions and avoid doing anything that is not relevant when
14040	* executing a nested-guest (e.g., Mesa driver hacks).
14041	*/
14042	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14043	{
14044	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
14045
14046	/*
14047	* If this VM-exit occurred while delivering an event through the guest IDT, take
14048	* action based on the return code and additional hints (e.g. for page-faults)
14049	* that will be updated in the VMX transient structure.
14050	*/
14051	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14052	if (rcStrict == VINF_SUCCESS)
14053	{
14054	/*
14055	* If an exception caused a VM-exit due to delivery of an event, the original
14056	* event may have to be re-injected into the guest. We shall reinject it and
14057	* continue guest execution. However, page-fault is a complicated case and
14058	* needs additional processing done in hmR0VmxExitXcptPF().
14059	*/
14060	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14061	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14062	if ( !pVCpu->hm.s.Event.fPending
14063	\|\| uVector == X86_XCPT_PF)
14064	{
14065	switch (uVector)
14066	{
14067	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
14068	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
14069	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
14070	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
14071	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
14072	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
14073	default:
14074	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
14075	}
14076	}
14077	/* else: inject pending event before resuming guest execution. */
14078	}
14079	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
14080	{
14081	Assert(pVCpu->hm.s.Event.fPending);
14082	rcStrict = VINF_SUCCESS;
14083	}
14084
14085	return rcStrict;
14086	}
14087	/** @} */
14088
14089
14090	/** @name VM-exit handlers.
14091	* @{
14092	*/
14093	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14094	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14095	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14096
14097	/**
14098	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14099	*/
14100	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14101	{
14102	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14103	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14104	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14105	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14106	return VINF_SUCCESS;
14107	return VINF_EM_RAW_INTERRUPT;
14108	}
14109
14110
14111	/**
14112	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14113	* VM-exit.
14114	*/
14115	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14116	{
14117	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14118	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14119
14120	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14121	AssertRCReturn(rc, rc);
14122
14123	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14124	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14125	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14126
14127	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14128	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14129	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14130	NOREF(pVmcsInfo);
14131
14132	VBOXSTRICTRC rcStrict;
14133	switch (uExitIntType)
14134	{
14135	/*
14136	* Host physical NMIs:
14137	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14138	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14139	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14140	*
14141	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14142	* See Intel spec. 27.5.5 "Updating Non-Register State".
14143	*/
14144	case VMX_EXIT_INT_INFO_TYPE_NMI:
14145	{
14146	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14147	break;
14148	}
14149
14150	/*
14151	* Privileged software exceptions (#DB from ICEBP),
14152	* Software exceptions (#BP and #OF),
14153	* Hardware exceptions:
14154	* Process the required exceptions and resume guest execution if possible.
14155	*/
14156	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14157	Assert(uVector == X86_XCPT_DB);
14158	RT_FALL_THRU();
14159	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14160	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14161	RT_FALL_THRU();
14162	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14163	{
14164	NOREF(uVector);
14165	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14166	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14167	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14168	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14169	AssertRCReturn(rc, rc);
14170
14171	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14172	break;
14173	}
14174
14175	default:
14176	{
14177	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14178	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14179	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14180	break;
14181	}
14182	}
14183
14184	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14185	return rcStrict;
14186	}
14187
14188
14189	/**
14190	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14191	*/
14192	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14193	{
14194	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14195
14196	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14197	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14198	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14199	AssertRCReturn(rc, rc);
14200
14201	/* Evaluate and deliver pending events and resume guest execution. */
14202	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14203	return VINF_SUCCESS;
14204	}
14205
14206
14207	/**
14208	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14209	*/
14210	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14211	{
14212	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14213
14214	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14215	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14216	{
14217	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14218	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14219	}
14220
14221	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14222
14223	/*
14224	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14225	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14226	*/
14227	uint32_t fIntrState;
14228	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14229	AssertRCReturn(rc, rc);
14230	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14231	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14232	{
14233	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14234	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14235
14236	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14237	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14238	AssertRCReturn(rc, rc);
14239	}
14240
14241	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14242	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14243	AssertRCReturn(rc, rc);
14244
14245	/* Evaluate and deliver pending events and resume guest execution. */
14246	return VINF_SUCCESS;
14247	}
14248
14249
14250	/**
14251	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14252	*/
14253	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14254	{
14255	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14256	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14257	}
14258
14259
14260	/**
14261	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14262	*/
14263	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14264	{
14265	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14266	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14267	}
14268
14269
14270	/**
14271	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14272	*/
14273	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14274	{
14275	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14276
14277	/*
14278	* Get the state we need and update the exit history entry.
14279	*/
14280	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14281	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14282	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14283	AssertRCReturn(rc, rc);
14284
14285	VBOXSTRICTRC rcStrict;
14286	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14287	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14288	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14289	if (!pExitRec)
14290	{
14291	/*
14292	* Regular CPUID instruction execution.
14293	*/
14294	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
14295	if (rcStrict == VINF_SUCCESS)
14296	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14297	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14298	{
14299	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14300	rcStrict = VINF_SUCCESS;
14301	}
14302	}
14303	else
14304	{
14305	/*
14306	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14307	*/
14308	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14309	AssertRCReturn(rc2, rc2);
14310
14311	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14312	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14313
14314	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14315	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14316
14317	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14318	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14319	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14320	}
14321	return rcStrict;
14322	}
14323
14324
14325	/**
14326	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14327	*/
14328	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14329	{
14330	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14331
14332	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14333	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14334	AssertRCReturn(rc, rc);
14335
14336	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14337	return VINF_EM_RAW_EMULATE_INSTR;
14338
14339	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14340	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14341	}
14342
14343
14344	/**
14345	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14346	*/
14347	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14348	{
14349	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14350
14351	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14352	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14353	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14354	AssertRCReturn(rc, rc);
14355
14356	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
14357	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14358	{
14359	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14360	we must reset offsetting on VM-entry. See @bugref{6634}. */
14361	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14362	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14363	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14364	}
14365	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14366	{
14367	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14368	rcStrict = VINF_SUCCESS;
14369	}
14370	return rcStrict;
14371	}
14372
14373
14374	/**
14375	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14376	*/
14377	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14378	{
14379	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14380
14381	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14382	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14383	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14384	AssertRCReturn(rc, rc);
14385
14386	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
14387	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14388	{
14389	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14390	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14391	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14392	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14393	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14394	}
14395	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14396	{
14397	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14398	rcStrict = VINF_SUCCESS;
14399	}
14400	return rcStrict;
14401	}
14402
14403
14404	/**
14405	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14406	*/
14407	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14408	{
14409	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14410
14411	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14412	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14413	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14414	AssertRCReturn(rc, rc);
14415
14416	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14417	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14418	if (RT_LIKELY(rc == VINF_SUCCESS))
14419	{
14420	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14421	Assert(pVmxTransient->cbInstr == 2);
14422	}
14423	else
14424	{
14425	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14426	rc = VERR_EM_INTERPRETER;
14427	}
14428	return rc;
14429	}
14430
14431
14432	/**
14433	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14434	*/
14435	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14436	{
14437	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14438
14439	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14440	if (EMAreHypercallInstructionsEnabled(pVCpu))
14441	{
14442	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14443	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14444	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14445	AssertRCReturn(rc, rc);
14446
14447	/* Perform the hypercall. */
14448	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14449	if (rcStrict == VINF_SUCCESS)
14450	{
14451	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14452	AssertRCReturn(rc, rc);
14453	}
14454	else
14455	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14456	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14457	\|\| RT_FAILURE(rcStrict));
14458
14459	/* If the hypercall changes anything other than guest's general-purpose registers,
14460	we would need to reload the guest changed bits here before VM-entry. */
14461	}
14462	else
14463	Log4Func(("Hypercalls not enabled\n"));
14464
14465	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14466	if (RT_FAILURE(rcStrict))
14467	{
14468	hmR0VmxSetPendingXcptUD(pVCpu);
14469	rcStrict = VINF_SUCCESS;
14470	}
14471
14472	return rcStrict;
14473	}
14474
14475
14476	/**
14477	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14478	*/
14479	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14480	{
14481	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14482	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14483
14484	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14485	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
14486	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14487	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14488	AssertRCReturn(rc, rc);
14489
14490	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
14491
14492	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14494	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14495	{
14496	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14497	rcStrict = VINF_SUCCESS;
14498	}
14499	else
14500	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14501	VBOXSTRICTRC_VAL(rcStrict)));
14502	return rcStrict;
14503	}
14504
14505
14506	/**
14507	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14508	*/
14509	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14510	{
14511	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14512
14513	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14514	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14515	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14516	AssertRCReturn(rc, rc);
14517
14518	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
14519	if (rcStrict == VINF_SUCCESS)
14520	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14521	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14522	{
14523	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14524	rcStrict = VINF_SUCCESS;
14525	}
14526
14527	return rcStrict;
14528	}
14529
14530
14531	/**
14532	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14533	*/
14534	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14535	{
14536	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14537
14538	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14539	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14540	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14541	AssertRCReturn(rc, rc);
14542
14543	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
14544	if (RT_SUCCESS(rcStrict))
14545	{
14546	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14547	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14548	rcStrict = VINF_SUCCESS;
14549	}
14550
14551	return rcStrict;
14552	}
14553
14554
14555	/**
14556	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14557	* VM-exit.
14558	*/
14559	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14560	{
14561	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14562	return VINF_EM_RESET;
14563	}
14564
14565
14566	/**
14567	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14568	*/
14569	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14570	{
14571	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14572
14573	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14574	AssertRCReturn(rc, rc);
14575
14576	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14577	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14578	rc = VINF_SUCCESS;
14579	else
14580	rc = VINF_EM_HALT;
14581
14582	if (rc != VINF_SUCCESS)
14583	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14584	return rc;
14585	}
14586
14587
14588	/**
14589	* VM-exit handler for instructions that result in a \#UD exception delivered to
14590	* the guest.
14591	*/
14592	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14593	{
14594	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14595	hmR0VmxSetPendingXcptUD(pVCpu);
14596	return VINF_SUCCESS;
14597	}
14598
14599
14600	/**
14601	* VM-exit handler for expiry of the VMX-preemption timer.
14602	*/
14603	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14604	{
14605	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14606
14607	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14608	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14609
14610	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14611	PVM pVM = pVCpu->CTX_SUFF(pVM);
14612	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14613	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14614	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14615	}
14616
14617
14618	/**
14619	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14620	*/
14621	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14622	{
14623	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14624
14625	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14626	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14627	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14628	AssertRCReturn(rc, rc);
14629
14630	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
14631	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14632	: HM_CHANGED_RAISED_XCPT_MASK);
14633
14634	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14635	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14636
14637	return rcStrict;
14638	}
14639
14640
14641	/**
14642	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14643	*/
14644	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14645	{
14646	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14647	/** @todo Use VM-exit instruction information. */
14648	return VERR_EM_INTERPRETER;
14649	}
14650
14651
14652	/**
14653	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14654	* VM-exit.
14655	*/
14656	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14657	{
14658	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14659	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14660	AssertRCReturn(rc, rc);
14661
14662	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14663	if (RT_FAILURE(rc))
14664	return rc;
14665
14666	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14667	NOREF(uInvalidReason);
14668
14669	#ifdef VBOX_STRICT
14670	uint32_t fIntrState;
14671	RTHCUINTREG uHCReg;
14672	uint64_t u64Val;
14673	uint32_t u32Val;
14674	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14675	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14676	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14677	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14678	AssertRCReturn(rc, rc);
14679
14680	Log4(("uInvalidReason %u\n", uInvalidReason));
14681	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14682	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14683	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14684	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14685
14686	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
14687	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
14688	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
14689	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
14690	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
14691	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14692	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
14693	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
14694	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
14695	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14696	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14697	{
14698	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14699	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14700	}
14701
14702	hmR0DumpRegs(pVCpu);
14703	#endif
14704
14705	return VERR_VMX_INVALID_GUEST_STATE;
14706	}
14707
14708	/**
14709	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14710	*/
14711	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14712	{
14713	/*
14714	* Cummulative notes of all recognized but unexpected VM-exits.
14715	*
14716	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
14717	* nested-paging is used.
14718	*
14719	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14720	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14721	* this function (and thereby stop VM execution) for handling such instructions.
14722	*
14723	*
14724	* VMX_EXIT_INIT_SIGNAL:
14725	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14726	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14727	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14728	*
14729	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14730	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14731	* See Intel spec. "23.8 Restrictions on VMX operation".
14732	*
14733	* VMX_EXIT_SIPI:
14734	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14735	* activity state is used. We don't make use of it as our guests don't have direct
14736	* access to the host local APIC.
14737	*
14738	* See Intel spec. 25.3 "Other Causes of VM-exits".
14739	*
14740	* VMX_EXIT_IO_SMI:
14741	* VMX_EXIT_SMI:
14742	* This can only happen if we support dual-monitor treatment of SMI, which can be
14743	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14744	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14745	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14746	*
14747	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14748	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14749	*
14750	* VMX_EXIT_ERR_MSR_LOAD:
14751	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14752	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14753	* execution.
14754	*
14755	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14756	*
14757	* VMX_EXIT_ERR_MACHINE_CHECK:
14758	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14759	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14760	* #MC exception abort class exception is raised. We thus cannot assume a
14761	* reasonable chance of continuing any sort of execution and we bail.
14762	*
14763	* See Intel spec. 15.1 "Machine-check Architecture".
14764	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14765	*
14766	* VMX_EXIT_PML_FULL:
14767	* VMX_EXIT_VIRTUALIZED_EOI:
14768	* VMX_EXIT_APIC_WRITE:
14769	* We do not currently support any of these features and thus they are all unexpected
14770	* VM-exits.
14771	*
14772	* VMX_EXIT_GDTR_IDTR_ACCESS:
14773	* VMX_EXIT_LDTR_TR_ACCESS:
14774	* VMX_EXIT_RDRAND:
14775	* VMX_EXIT_RSM:
14776	* VMX_EXIT_VMFUNC:
14777	* VMX_EXIT_ENCLS:
14778	* VMX_EXIT_RDSEED:
14779	* VMX_EXIT_XSAVES:
14780	* VMX_EXIT_XRSTORS:
14781	* VMX_EXIT_UMWAIT:
14782	* VMX_EXIT_TPAUSE:
14783	* These VM-exits are -not- caused unconditionally by execution of the corresponding
14784	* instruction. Any VM-exit for these instructions indicate a hardware problem,
14785	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14786	*
14787	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14788	*/
14789	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14790	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14791	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14792	}
14793
14794
14795	/**
14796	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14797	*/
14798	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14799	{
14800	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14801
14802	/** @todo Optimize this: We currently drag in in the whole MSR state
14803	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14804	* MSRs required. That would require changes to IEM and possibly CPUM too.
14805	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14806	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14807	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14808	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14809	switch (idMsr)
14810	{
14811	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14812	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14813	}
14814
14815	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14816	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14817	AssertRCReturn(rc, rc);
14818
14819	Log4Func(("ecx=%#RX32\n", idMsr));
14820
14821	#ifdef VBOX_STRICT
14822	Assert(!pVmxTransient->fIsNestedGuest);
14823	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
14824	{
14825	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14826	&& idMsr != MSR_K6_EFER)
14827	{
14828	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14829	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14830	}
14831	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14832	{
14833	Assert(pVmcsInfo->pvMsrBitmap);
14834	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14835	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14836	{
14837	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14838	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14839	}
14840	}
14841	}
14842	#endif
14843
14844	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
14845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14846	if (rcStrict == VINF_SUCCESS)
14847	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14848	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14849	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14850	{
14851	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14852	rcStrict = VINF_SUCCESS;
14853	}
14854	else
14855	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14856
14857	return rcStrict;
14858	}
14859
14860
14861	/**
14862	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14863	*/
14864	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14865	{
14866	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14867
14868	/** @todo Optimize this: We currently drag in in the whole MSR state
14869	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14870	* MSRs required. That would require changes to IEM and possibly CPUM too.
14871	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14872	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14873	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14874
14875	/*
14876	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14877	* Although we don't need to fetch the base as it will be overwritten shortly, while
14878	* loading guest-state we would also load the entire segment register including limit
14879	* and attributes and thus we need to load them here.
14880	*/
14881	switch (idMsr)
14882	{
14883	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14884	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14885	}
14886
14887	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14888	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14889	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14890	AssertRCReturn(rc, rc);
14891
14892	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14893
14894	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14896
14897	if (rcStrict == VINF_SUCCESS)
14898	{
14899	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14900
14901	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14902	if ( idMsr == MSR_IA32_APICBASE
14903	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14904	&& idMsr <= MSR_IA32_X2APIC_END))
14905	{
14906	/*
14907	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14908	* When full APIC register virtualization is implemented we'll have to make
14909	* sure APIC state is saved from the VMCS before IEM changes it.
14910	*/
14911	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14912	}
14913	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14914	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14915	else if (idMsr == MSR_K6_EFER)
14916	{
14917	/*
14918	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14919	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14920	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14921	*/
14922	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14923	}
14924
14925	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
14926	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14927	{
14928	switch (idMsr)
14929	{
14930	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14931	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14932	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14933	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14934	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14935	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14936	default:
14937	{
14938	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14939	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14940	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14941	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14942	break;
14943	}
14944	}
14945	}
14946	#ifdef VBOX_STRICT
14947	else
14948	{
14949	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14950	switch (idMsr)
14951	{
14952	case MSR_IA32_SYSENTER_CS:
14953	case MSR_IA32_SYSENTER_EIP:
14954	case MSR_IA32_SYSENTER_ESP:
14955	case MSR_K8_FS_BASE:
14956	case MSR_K8_GS_BASE:
14957	{
14958	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14959	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14960	}
14961
14962	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14963	default:
14964	{
14965	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14966	{
14967	/* EFER MSR writes are always intercepted. */
14968	if (idMsr != MSR_K6_EFER)
14969	{
14970	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14971	idMsr));
14972	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14973	}
14974	}
14975
14976	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14977	{
14978	Assert(pVmcsInfo->pvMsrBitmap);
14979	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14980	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14981	{
14982	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14983	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14984	}
14985	}
14986	break;
14987	}
14988	}
14989	}
14990	#endif /* VBOX_STRICT */
14991	}
14992	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14993	{
14994	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14995	rcStrict = VINF_SUCCESS;
14996	}
14997	else
14998	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14999
15000	return rcStrict;
15001	}
15002
15003
15004	/**
15005	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15006	*/
15007	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15008	{
15009	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15010
15011	/** @todo The guest has likely hit a contended spinlock. We might want to
15012	* poke a schedule different guest VCPU. */
15013	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15014	if (RT_SUCCESS(rc))
15015	return VINF_EM_RAW_INTERRUPT;
15016
15017	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15018	return rc;
15019	}
15020
15021
15022	/**
15023	* VM-exit handler for when the TPR value is lowered below the specified
15024	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15025	*/
15026	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15027	{
15028	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15029	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15030
15031	/*
15032	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15033	* We'll re-evaluate pending interrupts and inject them before the next VM
15034	* entry so we can just continue execution here.
15035	*/
15036	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15037	return VINF_SUCCESS;
15038	}
15039
15040
15041	/**
15042	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15043	* VM-exit.
15044	*
15045	* @retval VINF_SUCCESS when guest execution can continue.
15046	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15047	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15048	* incompatible guest state for VMX execution (real-on-v86 case).
15049	*/
15050	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15051	{
15052	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15053	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15054
15055	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15056	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
15057	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15058	AssertRCReturn(rc, rc);
15059
15060	VBOXSTRICTRC rcStrict;
15061	PVM pVM = pVCpu->CTX_SUFF(pVM);
15062	uint64_t const uExitQual = pVmxTransient->uExitQual;
15063	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15064	switch (uAccessType)
15065	{
15066	/*
15067	* MOV to CRx.
15068	*/
15069	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15070	{
15071	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15072	AssertRCReturn(rc, rc);
15073
15074	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15075	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15076	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15077	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15078
15079	/*
15080	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15081	* - When nested paging isn't used.
15082	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15083	* - We are executing in the VM debug loop.
15084	*/
15085	Assert( iCrReg != 3
15086	\|\| !pVM->hm.s.fNestedPaging
15087	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15088	\|\| pVCpu->hm.s.fUsingDebugLoop);
15089
15090	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15091	Assert( iCrReg != 8
15092	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15093
15094	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15095	AssertMsg( rcStrict == VINF_SUCCESS
15096	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15097
15098	/*
15099	* This is a kludge for handling switches back to real mode when we try to use
15100	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15101	* deal with special selector values, so we have to return to ring-3 and run
15102	* there till the selector values are V86 mode compatible.
15103	*
15104	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15105	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15106	* this function.
15107	*/
15108	if ( iCrReg == 0
15109	&& rcStrict == VINF_SUCCESS
15110	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15111	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15112	&& (uOldCr0 & X86_CR0_PE)
15113	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15114	{
15115	/** @todo Check selectors rather than returning all the time. */
15116	Assert(!pVmxTransient->fIsNestedGuest);
15117	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15118	rcStrict = VINF_EM_RESCHEDULE_REM;
15119	}
15120	break;
15121	}
15122
15123	/*
15124	* MOV from CRx.
15125	*/
15126	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15127	{
15128	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15129	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15130
15131	/*
15132	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15133	* - When nested paging isn't used.
15134	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15135	* - We are executing in the VM debug loop.
15136	*/
15137	Assert( iCrReg != 3
15138	\|\| !pVM->hm.s.fNestedPaging
15139	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15140	\|\| pVCpu->hm.s.fUsingDebugLoop);
15141
15142	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15143	Assert( iCrReg != 8
15144	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15145
15146	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15147	break;
15148	}
15149
15150	/*
15151	* CLTS (Clear Task-Switch Flag in CR0).
15152	*/
15153	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15154	{
15155	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15156	break;
15157	}
15158
15159	/*
15160	* LMSW (Load Machine-Status Word into CR0).
15161	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15162	*/
15163	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15164	{
15165	RTGCPTR GCPtrEffDst;
15166	uint8_t const cbInstr = pVmxTransient->cbInstr;
15167	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15168	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15169	if (fMemOperand)
15170	{
15171	rc = hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
15172	AssertRCReturn(rc, rc);
15173	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15174	}
15175	else
15176	GCPtrEffDst = NIL_RTGCPTR;
15177	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15178	break;
15179	}
15180
15181	default:
15182	{
15183	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15184	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15185	}
15186	}
15187
15188	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15189	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15190	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15191
15192	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15193	NOREF(pVM);
15194	return rcStrict;
15195	}
15196
15197
15198	/**
15199	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15200	* VM-exit.
15201	*/
15202	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15203	{
15204	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15205	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15206
15207	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15208	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15209	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
15210	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15211	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15212	\| CPUMCTX_EXTRN_EFER);
15213	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15214	AssertRCReturn(rc, rc);
15215
15216	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15217	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15218	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15219	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15220	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15221	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15222	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15223	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15224
15225	/*
15226	* Update exit history to see if this exit can be optimized.
15227	*/
15228	VBOXSTRICTRC rcStrict;
15229	PCEMEXITREC pExitRec = NULL;
15230	if ( !fGstStepping
15231	&& !fDbgStepping)
15232	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15233	!fIOString
15234	? !fIOWrite
15235	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15236	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15237	: !fIOWrite
15238	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15239	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15240	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15241	if (!pExitRec)
15242	{
15243	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15244	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15245
15246	uint32_t const cbValue = s_aIOSizes[uIOSize];
15247	uint32_t const cbInstr = pVmxTransient->cbInstr;
15248	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15249	PVM pVM = pVCpu->CTX_SUFF(pVM);
15250	if (fIOString)
15251	{
15252	/*
15253	* INS/OUTS - I/O String instruction.
15254	*
15255	* Use instruction-information if available, otherwise fall back on
15256	* interpreting the instruction.
15257	*/
15258	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15259	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15260	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15261	if (fInsOutsInfo)
15262	{
15263	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15264	AssertRCReturn(rc2, rc2);
15265	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15266	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15267	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15268	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15269	if (fIOWrite)
15270	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15271	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15272	else
15273	{
15274	/*
15275	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15276	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15277	* See Intel Instruction spec. for "INS".
15278	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15279	*/
15280	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15281	}
15282	}
15283	else
15284	rcStrict = IEMExecOne(pVCpu);
15285
15286	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15287	fUpdateRipAlready = true;
15288	}
15289	else
15290	{
15291	/*
15292	* IN/OUT - I/O instruction.
15293	*/
15294	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15295	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15296	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15297	if (fIOWrite)
15298	{
15299	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15301	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15302	&& !pCtx->eflags.Bits.u1TF)
15303	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15304	}
15305	else
15306	{
15307	uint32_t u32Result = 0;
15308	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15309	if (IOM_SUCCESS(rcStrict))
15310	{
15311	/* Save result of I/O IN instr. in AL/AX/EAX. */
15312	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15313	}
15314	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15315	&& !pCtx->eflags.Bits.u1TF)
15316	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15318	}
15319	}
15320
15321	if (IOM_SUCCESS(rcStrict))
15322	{
15323	if (!fUpdateRipAlready)
15324	{
15325	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15326	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15327	}
15328
15329	/*
15330	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15331	* while booting Fedora 17 64-bit guest.
15332	*
15333	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15334	*/
15335	if (fIOString)
15336	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15337
15338	/*
15339	* If any I/O breakpoints are armed, we need to check if one triggered
15340	* and take appropriate action.
15341	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15342	*/
15343	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15344	AssertRCReturn(rc, rc);
15345
15346	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15347	* execution engines about whether hyper BPs and such are pending. */
15348	uint32_t const uDr7 = pCtx->dr[7];
15349	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15350	&& X86_DR7_ANY_RW_IO(uDr7)
15351	&& (pCtx->cr4 & X86_CR4_DE))
15352	\|\| DBGFBpIsHwIoArmed(pVM)))
15353	{
15354	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15355
15356	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15357	VMMRZCallRing3Disable(pVCpu);
15358	HM_DISABLE_PREEMPT(pVCpu);
15359
15360	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15361
15362	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15363	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15364	{
15365	/* Raise #DB. */
15366	if (fIsGuestDbgActive)
15367	ASMSetDR6(pCtx->dr[6]);
15368	if (pCtx->dr[7] != uDr7)
15369	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15370
15371	hmR0VmxSetPendingXcptDB(pVCpu);
15372	}
15373	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15374	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15375	else if ( rcStrict2 != VINF_SUCCESS
15376	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15377	rcStrict = rcStrict2;
15378	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15379
15380	HM_RESTORE_PREEMPT();
15381	VMMRZCallRing3Enable(pVCpu);
15382	}
15383	}
15384
15385	#ifdef VBOX_STRICT
15386	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15387	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15388	Assert(!fIOWrite);
15389	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15390	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15391	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15392	Assert(fIOWrite);
15393	else
15394	{
15395	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15396	* statuses, that the VMM device and some others may return. See
15397	* IOM_SUCCESS() for guidance. */
15398	AssertMsg( RT_FAILURE(rcStrict)
15399	\|\| rcStrict == VINF_SUCCESS
15400	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15401	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15402	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15403	\|\| rcStrict == VINF_EM_RAW_TO_R3
15404	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15405	# endif
15406	}
15407	#endif
15408	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15409	}
15410	else
15411	{
15412	/*
15413	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15414	*/
15415	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15416	AssertRCReturn(rc2, rc2);
15417	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15418	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15419	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15420	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15421	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15422	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15423
15424	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15425	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15426
15427	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15428	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15429	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15430	}
15431	return rcStrict;
15432	}
15433
15434
15435	/**
15436	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15437	* VM-exit.
15438	*/
15439	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15440	{
15441	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15442
15443	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15444	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
15445	AssertRCReturn(rc, rc);
15446	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15447	{
15448	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15449	AssertRCReturn(rc, rc);
15450	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15451	{
15452	uint32_t uErrCode;
15453	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15454	{
15455	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15456	AssertRCReturn(rc, rc);
15457	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15458	}
15459	else
15460	uErrCode = 0;
15461
15462	RTGCUINTPTR GCPtrFaultAddress;
15463	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15464	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15465	else
15466	GCPtrFaultAddress = 0;
15467
15468	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15469	AssertRCReturn(rc, rc);
15470
15471	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15472	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
15473
15474	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15475	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15476	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15477	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15478	}
15479	}
15480
15481	/* Fall back to the interpreter to emulate the task-switch. */
15482	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15483	return VERR_EM_INTERPRETER;
15484	}
15485
15486
15487	/**
15488	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15489	*/
15490	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15491	{
15492	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15493
15494	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15495	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15496	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15497	AssertRCReturn(rc, rc);
15498	return VINF_EM_DBG_STEPPED;
15499	}
15500
15501
15502	/**
15503	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15504	*/
15505	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15506	{
15507	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15508	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15509
15510	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15511	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15512	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15513	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15514	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15515	AssertRCReturn(rc, rc);
15516
15517	/*
15518	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15519	*/
15520	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15521	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15522	{
15523	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15524	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15525	{
15526	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15527	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15528	}
15529	}
15530	else
15531	{
15532	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15533	return rcStrict;
15534	}
15535
15536	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15537	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15538	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15539	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
15540	AssertRCReturn(rc, rc);
15541
15542	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15543	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15544	switch (uAccessType)
15545	{
15546	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15547	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15548	{
15549	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15550	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15551	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15552
15553	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15554	GCPhys &= PAGE_BASE_GC_MASK;
15555	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15556	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15557	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15558
15559	PVM pVM = pVCpu->CTX_SUFF(pVM);
15560	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15561	rcStrict = IOMMMIOPhysHandler(pVM, pVCpu,
15562	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15563	CPUMCTX2CORE(pCtx), GCPhys);
15564	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15565	if ( rcStrict == VINF_SUCCESS
15566	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15567	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15568	{
15569	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15570	\| HM_CHANGED_GUEST_APIC_TPR);
15571	rcStrict = VINF_SUCCESS;
15572	}
15573	break;
15574	}
15575
15576	default:
15577	{
15578	Log4Func(("uAccessType=%#x\n", uAccessType));
15579	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15580	break;
15581	}
15582	}
15583
15584	if (rcStrict != VINF_SUCCESS)
15585	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15586	return rcStrict;
15587	}
15588
15589
15590	/**
15591	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15592	* VM-exit.
15593	*/
15594	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15595	{
15596	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15597	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15598
15599	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15600	if (!pVmxTransient->fIsNestedGuest)
15601	{
15602	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15603	if (pVmxTransient->fWasGuestDebugStateActive)
15604	{
15605	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15606	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15607	}
15608
15609	if ( !pVCpu->hm.s.fSingleInstruction
15610	&& !pVmxTransient->fWasHyperDebugStateActive)
15611	{
15612	Assert(!DBGFIsStepping(pVCpu));
15613	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15614
15615	/* Don't intercept MOV DRx any more. */
15616	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15617	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15618	AssertRCReturn(rc, rc);
15619
15620	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15621	VMMRZCallRing3Disable(pVCpu);
15622	HM_DISABLE_PREEMPT(pVCpu);
15623
15624	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15625	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15626	Assert(CPUMIsGuestDebugStateActive(pVCpu));
15627
15628	HM_RESTORE_PREEMPT();
15629	VMMRZCallRing3Enable(pVCpu);
15630
15631	#ifdef VBOX_WITH_STATISTICS
15632	rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
15633	AssertRCReturn(rc, rc);
15634	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15635	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15636	else
15637	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15638	#endif
15639	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15640	return VINF_SUCCESS;
15641	}
15642	}
15643
15644	/*
15645	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15646	* The EFER MSR is always up-to-date.
15647	* Update the segment registers and DR7 from the CPU.
15648	*/
15649	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15650	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
15651	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15652	AssertRCReturn(rc, rc);
15653	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15654
15655	PVM pVM = pVCpu->CTX_SUFF(pVM);
15656	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15657	{
15658	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15659	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15660	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15661	if (RT_SUCCESS(rc))
15662	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15663	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15664	}
15665	else
15666	{
15667	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15668	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15669	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15670	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15671	}
15672
15673	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15674	if (RT_SUCCESS(rc))
15675	{
15676	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15677	AssertRCReturn(rc2, rc2);
15678	return VINF_SUCCESS;
15679	}
15680	return rc;
15681	}
15682
15683
15684	/**
15685	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15686	* Conditional VM-exit.
15687	*/
15688	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15689	{
15690	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15691	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15692
15693	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15694	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15695	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15696	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15697	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15698	AssertRCReturn(rc, rc);
15699
15700	/*
15701	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15702	*/
15703	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15704	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15705	{
15706	/*
15707	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15708	* instruction emulation to inject the original event. Otherwise, injecting the original event
15709	* using hardware-assisted VMX would would trigger the same EPT misconfig VM-exit again.
15710	*/
15711	if (!pVCpu->hm.s.Event.fPending)
15712	{ /* likely */ }
15713	else
15714	{
15715	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15716	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15717	/** @todo NSTVMX: Think about how this should be handled. */
15718	if (pVmxTransient->fIsNestedGuest)
15719	return VERR_VMX_IPE_3;
15720	#endif
15721	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15722	}
15723	}
15724	else
15725	{
15726	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15727	return rcStrict;
15728	}
15729
15730	/*
15731	* Get sufficent state and update the exit history entry.
15732	*/
15733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15734	rc = hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15735	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15736	AssertRCReturn(rc, rc);
15737
15738	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15739	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15740	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15741	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15742	if (!pExitRec)
15743	{
15744	/*
15745	* If we succeed, resume guest execution.
15746	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15747	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15748	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15749	* weird case. See @bugref{6043}.
15750	*/
15751	PVM pVM = pVCpu->CTX_SUFF(pVM);
15752	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15753	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15754	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15755	if ( rcStrict == VINF_SUCCESS
15756	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15757	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15758	{
15759	/* Successfully handled MMIO operation. */
15760	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15761	\| HM_CHANGED_GUEST_APIC_TPR);
15762	rcStrict = VINF_SUCCESS;
15763	}
15764	}
15765	else
15766	{
15767	/*
15768	* Frequent exit or something needing probing. Call EMHistoryExec.
15769	*/
15770	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15771	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15772
15773	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15774	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15775
15776	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15777	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15778	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15779	}
15780	return rcStrict;
15781	}
15782
15783
15784	/**
15785	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15786	* VM-exit.
15787	*/
15788	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15789	{
15790	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15791	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15792
15793	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
15794	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15795	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15796	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15797	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15798	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15799	AssertRCReturn(rc, rc);
15800
15801	/*
15802	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15803	*/
15804	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15805	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15806	{
15807	/*
15808	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
15809	* we shall resolve the nested #PF and re-inject the original event.
15810	*/
15811	if (pVCpu->hm.s.Event.fPending)
15812	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
15813	}
15814	else
15815	{
15816	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15817	return rcStrict;
15818	}
15819
15820	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15821	rc = hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15822	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15823	AssertRCReturn(rc, rc);
15824
15825	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15826	uint64_t const uExitQual = pVmxTransient->uExitQual;
15827	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
15828
15829	RTGCUINT uErrorCode = 0;
15830	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15831	uErrorCode \|= X86_TRAP_PF_ID;
15832	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15833	uErrorCode \|= X86_TRAP_PF_RW;
15834	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15835	uErrorCode \|= X86_TRAP_PF_P;
15836
15837	PVM pVM = pVCpu->CTX_SUFF(pVM);
15838	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15839	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
15840
15841	/*
15842	* Handle the pagefault trap for the nested shadow table.
15843	*/
15844	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15845	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15846	TRPMResetTrap(pVCpu);
15847
15848	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15849	if ( rcStrict == VINF_SUCCESS
15850	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15851	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15852	{
15853	/* Successfully synced our nested page tables. */
15854	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15855	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15856	return VINF_SUCCESS;
15857	}
15858
15859	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15860	return rcStrict;
15861	}
15862
15863
15864	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15865	/**
15866	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15867	*/
15868	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15869	{
15870	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15871
15872	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15873	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15874	\| CPUMCTX_EXTRN_HWVIRT
15875	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15876	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15877	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
15878	AssertRCReturn(rc, rc);
15879
15880	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15881
15882	VMXVEXITINFO ExitInfo;
15883	RT_ZERO(ExitInfo);
15884	ExitInfo.uReason = pVmxTransient->uExitReason;
15885	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15886	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15887	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15888	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15889
15890	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15891	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15892	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15893	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15894	{
15895	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15896	rcStrict = VINF_SUCCESS;
15897	}
15898	return rcStrict;
15899	}
15900
15901
15902	/**
15903	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15904	*/
15905	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15906	{
15907	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15908
15909	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15910	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15911	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15912	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15913	AssertRCReturn(rc, rc);
15914
15915	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15916
15917	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
15918	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15919	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
15920	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15921	{
15922	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15923	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
15924	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15925	}
15926	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15927	return rcStrict;
15928	}
15929
15930
15931	/**
15932	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15933	*/
15934	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15935	{
15936	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15937
15938	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15939	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15940	\| CPUMCTX_EXTRN_HWVIRT
15941	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15942	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15943	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
15944	AssertRCReturn(rc, rc);
15945
15946	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15947
15948	VMXVEXITINFO ExitInfo;
15949	RT_ZERO(ExitInfo);
15950	ExitInfo.uReason = pVmxTransient->uExitReason;
15951	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15952	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15953	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15954	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15955
15956	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15957	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15958	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15959	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15960	{
15961	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15962	rcStrict = VINF_SUCCESS;
15963	}
15964	return rcStrict;
15965	}
15966
15967
15968	/**
15969	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15970	*/
15971	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15972	{
15973	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15974
15975	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15976	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15977	\| CPUMCTX_EXTRN_HWVIRT
15978	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15979	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15980	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
15981	AssertRCReturn(rc, rc);
15982
15983	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15984
15985	VMXVEXITINFO ExitInfo;
15986	RT_ZERO(ExitInfo);
15987	ExitInfo.uReason = pVmxTransient->uExitReason;
15988	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15989	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15990	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15991	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15992
15993	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15994	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15995	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15996	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15997	{
15998	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15999	rcStrict = VINF_SUCCESS;
16000	}
16001	return rcStrict;
16002	}
16003
16004
16005	/**
16006	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16007	*/
16008	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16009	{
16010	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16011
16012	/*
16013	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16014	* thus might not need to import the shadow VMCS state, it's safer just in case
16015	* code elsewhere dares look at unsynced VMCS fields.
16016	*/
16017	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16018	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16019	\| CPUMCTX_EXTRN_HWVIRT
16020	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16021	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16022	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
16023	AssertRCReturn(rc, rc);
16024
16025	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16026
16027	VMXVEXITINFO ExitInfo;
16028	RT_ZERO(ExitInfo);
16029	ExitInfo.uReason = pVmxTransient->uExitReason;
16030	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16031	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16032	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16033	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16034	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16035
16036	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16037	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16038	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16039	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16040	{
16041	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16042	rcStrict = VINF_SUCCESS;
16043	}
16044	return rcStrict;
16045	}
16046
16047
16048	/**
16049	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16050	*/
16051	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16052	{
16053	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16054
16055	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16056	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16057	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16058	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16059	AssertRCReturn(rc, rc);
16060
16061	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16062
16063	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16064	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16065	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16066	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16067	{
16068	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16069	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16070	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16071	}
16072	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16073	return rcStrict;
16074	}
16075
16076
16077	/**
16078	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16079	*/
16080	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16081	{
16082	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16083
16084	/*
16085	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16086	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16087	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16088	*/
16089	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16090	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16091	\| CPUMCTX_EXTRN_HWVIRT
16092	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16093	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16094	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
16095	AssertRCReturn(rc, rc);
16096
16097	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16098
16099	VMXVEXITINFO ExitInfo;
16100	RT_ZERO(ExitInfo);
16101	ExitInfo.uReason = pVmxTransient->uExitReason;
16102	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16103	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16104	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16105	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16106	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16107
16108	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16109	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16110	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16111	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16112	{
16113	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16114	rcStrict = VINF_SUCCESS;
16115	}
16116	return rcStrict;
16117	}
16118
16119
16120	/**
16121	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16122	*/
16123	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16124	{
16125	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16126
16127	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16128	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16129	\| CPUMCTX_EXTRN_HWVIRT
16130	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16131	AssertRCReturn(rc, rc);
16132
16133	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16134
16135	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16136	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16137	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16138	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16139	{
16140	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16141	rcStrict = VINF_SUCCESS;
16142	}
16143	return rcStrict;
16144	}
16145
16146
16147	/**
16148	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16149	*/
16150	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16151	{
16152	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16153
16154	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16155	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16156	\| CPUMCTX_EXTRN_HWVIRT
16157	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16158	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16159	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
16160	AssertRCReturn(rc, rc);
16161
16162	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16163
16164	VMXVEXITINFO ExitInfo;
16165	RT_ZERO(ExitInfo);
16166	ExitInfo.uReason = pVmxTransient->uExitReason;
16167	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16168	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16169	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16170	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16171
16172	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16173	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16174	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16175	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16176	{
16177	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16178	rcStrict = VINF_SUCCESS;
16179	}
16180	return rcStrict;
16181	}
16182
16183
16184	/**
16185	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16186	*/
16187	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16188	{
16189	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16190
16191	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16192	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16193	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16194	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16195	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
16196	AssertRCReturn(rc, rc);
16197
16198	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16199
16200	VMXVEXITINFO ExitInfo;
16201	RT_ZERO(ExitInfo);
16202	ExitInfo.uReason = pVmxTransient->uExitReason;
16203	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16204	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16205	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16206	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16207
16208	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16209	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16210	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16211	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16212	{
16213	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16214	rcStrict = VINF_SUCCESS;
16215	}
16216	return rcStrict;
16217	}
16218	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16219	/** @} */
16220
16221
16222	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16223	/** @name Nested-guest VM-exit handlers.
16224	* @{
16225	*/
16226	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16227	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16228	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16229
16230	/**
16231	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16232	* Conditional VM-exit.
16233	*/
16234	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16235	{
16236	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16237
16238	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16239	AssertRCReturn(rc, rc);
16240
16241	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16242	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16243	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16244
16245	switch (uExitIntType)
16246	{
16247	/*
16248	* Physical NMIs:
16249	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16250	*/
16251	case VMX_EXIT_INT_INFO_TYPE_NMI:
16252	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16253
16254	/*
16255	* Hardware exceptions,
16256	* Software exceptions,
16257	* Privileged software exceptions:
16258	* Figure out if the exception must be delivered to the guest or the nested-guest.
16259	*/
16260	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16261	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16262	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16263	{
16264	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16265	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16266	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16267	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16268	AssertRCReturn(rc, rc);
16269
16270	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16271	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16272	pVmxTransient->uExitIntErrorCode);
16273	if (fIntercept)
16274	{
16275	/* Exit qualification is required for debug and page-fault exceptions. */
16276	rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
16277	AssertRCReturn(rc, rc);
16278
16279	/*
16280	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16281	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16282	* length. However, if delivery of a software interrupt, software exception or privileged
16283	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16284	*/
16285	VMXVEXITINFO ExitInfo;
16286	RT_ZERO(ExitInfo);
16287	ExitInfo.uReason = pVmxTransient->uExitReason;
16288	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16289	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16290
16291	VMXVEXITEVENTINFO ExitEventInfo;
16292	RT_ZERO(ExitEventInfo);
16293	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16294	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16295	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16296	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16297
16298	#ifdef DEBUG_ramshankar
16299	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
16300	Log4Func(("cs:rip=%#04x:%#RX64 %s err_code=%#x exit_qual=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
16301	VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo) ? "#PF" : "Unk",
16302	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16303	Log4Func(("idt_info=%#RX64 (%s) idt_errcode=%#RX32\n", pVmxTransient->uIdtVectoringInfo,
16304	VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo) ? "Valid" : "Invalid",
16305	pVmxTransient->uIdtVectoringErrorCode));
16306	#endif
16307	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16308	}
16309
16310	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16311	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16312	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16313	}
16314
16315	/*
16316	* Software interrupts:
16317	* VM-exits cannot be caused by software interrupts.
16318	*
16319	* External interrupts:
16320	* This should only happen when "acknowledge external interrupts on VM-exit"
16321	* control is set. However, we never set this when executing a guest or
16322	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16323	* the guest.
16324	*/
16325	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16326	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16327	default:
16328	{
16329	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16330	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16331	}
16332	}
16333	}
16334
16335
16336	/**
16337	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16338	* Unconditional VM-exit.
16339	*/
16340	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16341	{
16342	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16343	return IEMExecVmxVmexitTripleFault(pVCpu);
16344	}
16345
16346
16347	/**
16348	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16349	*/
16350	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16351	{
16352	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16353
16354	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16355	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16356	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16357	}
16358
16359
16360	/**
16361	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16362	*/
16363	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16364	{
16365	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16366
16367	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16368	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16369	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16370	}
16371
16372
16373	/**
16374	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16375	* Unconditional VM-exit.
16376	*/
16377	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16378	{
16379	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16380
16381	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
16382	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16383	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16384	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16385	AssertRCReturn(rc, rc);
16386
16387	VMXVEXITINFO ExitInfo;
16388	RT_ZERO(ExitInfo);
16389	ExitInfo.uReason = pVmxTransient->uExitReason;
16390	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16391	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16392
16393	VMXVEXITEVENTINFO ExitEventInfo;
16394	RT_ZERO(ExitEventInfo);
16395	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16396	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16397	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16398	}
16399
16400
16401	/**
16402	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16403	*/
16404	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16405	{
16406	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16407
16408	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16409	{
16410	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16411	AssertRCReturn(rc, rc);
16412	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16413	}
16414	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16415	}
16416
16417
16418	/**
16419	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16420	*/
16421	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16422	{
16423	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16424
16425	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16426	{
16427	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16428	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
16429	AssertRCReturn(rc, rc);
16430
16431	VMXVEXITINFO ExitInfo;
16432	RT_ZERO(ExitInfo);
16433	ExitInfo.uReason = pVmxTransient->uExitReason;
16434	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16435	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16436	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16437	}
16438	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16439	}
16440
16441
16442	/**
16443	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16444	*/
16445	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16446	{
16447	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16448
16449	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16450	{
16451	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16452	AssertRCReturn(rc, rc);
16453	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16454	}
16455	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16456	}
16457
16458
16459	/**
16460	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16461	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16462	*/
16463	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16464	{
16465	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16466
16467	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16468	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16469
16470	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16471	AssertRCReturn(rc, rc);
16472
16473	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16474	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16475	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16476
16477	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16478	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16479	u64VmcsField &= UINT64_C(0xffffffff);
16480
16481	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16482	{
16483	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16484	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
16485	AssertRCReturn(rc, rc);
16486
16487	VMXVEXITINFO ExitInfo;
16488	RT_ZERO(ExitInfo);
16489	ExitInfo.uReason = pVmxTransient->uExitReason;
16490	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16491	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16492	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16493	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16494	}
16495
16496	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16497	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16498	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16499	}
16500
16501
16502	/**
16503	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16504	*/
16505	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16506	{
16507	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16508
16509	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16510	{
16511	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16512	AssertRCReturn(rc, rc);
16513	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16514	}
16515
16516	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16517	}
16518
16519
16520	/**
16521	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16522	* Conditional VM-exit.
16523	*/
16524	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16525	{
16526	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16527
16528	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
16529	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16530	AssertRCReturn(rc, rc);
16531
16532	VBOXSTRICTRC rcStrict;
16533	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16534	switch (uAccessType)
16535	{
16536	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16537	{
16538	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16539	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16540	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16541	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16542
16543	bool fIntercept;
16544	switch (iCrReg)
16545	{
16546	case 0:
16547	case 4:
16548	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16549	break;
16550
16551	case 3:
16552	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16553	break;
16554
16555	case 8:
16556	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16557	break;
16558
16559	default:
16560	fIntercept = false;
16561	break;
16562	}
16563	if (fIntercept)
16564	{
16565	VMXVEXITINFO ExitInfo;
16566	RT_ZERO(ExitInfo);
16567	ExitInfo.uReason = pVmxTransient->uExitReason;
16568	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16569	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16570	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16571	}
16572	else
16573	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16574	break;
16575	}
16576
16577	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16578	{
16579	/*
16580	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16581	* CR2 reads do not cause a VM-exit.
16582	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16583	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16584	*/
16585	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16586	if ( iCrReg == 3
16587	\|\| iCrReg == 8)
16588	{
16589	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16590	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16591	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16592	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16593	{
16594	VMXVEXITINFO ExitInfo;
16595	RT_ZERO(ExitInfo);
16596	ExitInfo.uReason = pVmxTransient->uExitReason;
16597	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16598	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16599	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16600	}
16601	else
16602	{
16603	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16604	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16605	}
16606	}
16607	else
16608	{
16609	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16610	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16611	}
16612	break;
16613	}
16614
16615	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16616	{
16617	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16618	Assert(pVmcsNstGst);
16619	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16620	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16621	if ( (uGstHostMask & X86_CR0_TS)
16622	&& (uReadShadow & X86_CR0_TS))
16623	{
16624	VMXVEXITINFO ExitInfo;
16625	RT_ZERO(ExitInfo);
16626	ExitInfo.uReason = pVmxTransient->uExitReason;
16627	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16628	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16629	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16630	}
16631	else
16632	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
16633	break;
16634	}
16635
16636	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16637	{
16638	RTGCPTR GCPtrEffDst;
16639	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16640	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16641	if (fMemOperand)
16642	{
16643	rc = hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16644	AssertRCReturn(rc, rc);
16645	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16646	}
16647	else
16648	GCPtrEffDst = NIL_RTGCPTR;
16649
16650	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16651	{
16652	VMXVEXITINFO ExitInfo;
16653	RT_ZERO(ExitInfo);
16654	ExitInfo.uReason = pVmxTransient->uExitReason;
16655	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16656	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16657	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16658	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16659	}
16660	else
16661	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
16662	break;
16663	}
16664
16665	default:
16666	{
16667	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16668	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16669	}
16670	}
16671
16672	if (rcStrict == VINF_IEM_RAISED_XCPT)
16673	{
16674	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16675	rcStrict = VINF_SUCCESS;
16676	}
16677	return rcStrict;
16678	}
16679
16680
16681	/**
16682	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16683	* Conditional VM-exit.
16684	*/
16685	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16686	{
16687	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16688
16689	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16690	{
16691	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
16692	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16693	AssertRCReturn(rc, rc);
16694
16695	VMXVEXITINFO ExitInfo;
16696	RT_ZERO(ExitInfo);
16697	ExitInfo.uReason = pVmxTransient->uExitReason;
16698	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16699	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16700	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16701	}
16702	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16703	}
16704
16705
16706	/**
16707	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16708	* Conditional VM-exit.
16709	*/
16710	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16711	{
16712	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16713
16714	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
16715	AssertRCReturn(rc, rc);
16716
16717	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16718	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16719	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16720
16721	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16722	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16723	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16724	{
16725	/*
16726	* IN/OUT instruction:
16727	* - Provides VM-exit instruction length.
16728	*
16729	* INS/OUTS instruction:
16730	* - Provides VM-exit instruction length.
16731	* - Provides Guest-linear address.
16732	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16733	*/
16734	PVM pVM = pVCpu->CTX_SUFF(pVM);
16735	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16736	AssertRCReturn(rc, rc);
16737
16738	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16739	pVmxTransient->ExitInstrInfo.u = 0;
16740	pVmxTransient->uGuestLinearAddr = 0;
16741
16742	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16743	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16744	if (fIOString)
16745	{
16746	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16747	if (fVmxInsOutsInfo)
16748	{
16749	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16750	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16751	}
16752	}
16753	AssertRCReturn(rc, rc);
16754
16755	VMXVEXITINFO ExitInfo;
16756	RT_ZERO(ExitInfo);
16757	ExitInfo.uReason = pVmxTransient->uExitReason;
16758	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16759	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16760	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16761	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16762	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16763	}
16764	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16765	}
16766
16767
16768	/**
16769	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16770	*/
16771	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16772	{
16773	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16774
16775	uint32_t fMsrpm;
16776	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16777	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16778	else
16779	fMsrpm = VMXMSRPM_EXIT_RD;
16780
16781	if (fMsrpm & VMXMSRPM_EXIT_RD)
16782	{
16783	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16784	AssertRCReturn(rc, rc);
16785	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16786	}
16787	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16788	}
16789
16790
16791	/**
16792	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16793	*/
16794	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16795	{
16796	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16797
16798	uint32_t fMsrpm;
16799	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16800	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16801	else
16802	fMsrpm = VMXMSRPM_EXIT_WR;
16803
16804	if (fMsrpm & VMXMSRPM_EXIT_WR)
16805	{
16806	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16807	AssertRCReturn(rc, rc);
16808	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16809	}
16810	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16811	}
16812
16813
16814	/**
16815	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16816	*/
16817	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16818	{
16819	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16820
16821	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16822	{
16823	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16824	AssertRCReturn(rc, rc);
16825	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16826	}
16827	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16828	}
16829
16830
16831	/**
16832	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16833	* VM-exit.
16834	*/
16835	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16836	{
16837	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16838
16839	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16840	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16841	}
16842
16843
16844	/**
16845	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16846	*/
16847	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16848	{
16849	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16850
16851	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16852	{
16853	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16854	AssertRCReturn(rc, rc);
16855	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16856	}
16857	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16858	}
16859
16860
16861	/**
16862	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16863	*/
16864	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16865	{
16866	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16867
16868	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16869	* PAUSE when executing a nested-guest? If it does not, we would not need
16870	* to check for the intercepts here. Just call VM-exit... */
16871
16872	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16873	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16874	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16875	{
16876	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16877	AssertRCReturn(rc, rc);
16878	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16879	}
16880	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16881	}
16882
16883
16884	/**
16885	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16886	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16887	*/
16888	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16889	{
16890	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16891
16892	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16893	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16894	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16895	}
16896
16897
16898	/**
16899	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16900	* VM-exit.
16901	*/
16902	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16903	{
16904	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16905
16906	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16907	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16908	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16909	AssertRCReturn(rc, rc);
16910
16911	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16912	rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
16913	AssertRCReturn(rc, rc);
16914
16915	VMXVEXITINFO ExitInfo;
16916	RT_ZERO(ExitInfo);
16917	ExitInfo.uReason = pVmxTransient->uExitReason;
16918	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16919	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16920
16921	VMXVEXITEVENTINFO ExitEventInfo;
16922	RT_ZERO(ExitEventInfo);
16923	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16924	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16925	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
16926	}
16927
16928
16929	/**
16930	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
16931	* Conditional VM-exit.
16932	*/
16933	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16934	{
16935	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16936
16937	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
16938	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
16939	AssertRCReturn(rc, rc);
16940
16941	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16942	}
16943
16944
16945	/**
16946	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
16947	* Conditional VM-exit.
16948	*/
16949	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16950	{
16951	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16952
16953	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
16954	int rc = hmR0VmxReadExitQualVmcs(pVmxTransient);
16955	AssertRCReturn(rc, rc);
16956
16957	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16958	}
16959
16960
16961	/**
16962	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16963	*/
16964	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16965	{
16966	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16967
16968	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16969	{
16970	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16971	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16972	AssertRCReturn(rc, rc);
16973	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16974	}
16975	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16976	}
16977
16978
16979	/**
16980	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16981	*/
16982	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16983	{
16984	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16985
16986	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16987	{
16988	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16989	AssertRCReturn(rc, rc);
16990	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16991	}
16992	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16993	}
16994
16995
16996	/**
16997	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16998	*/
16999	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17000	{
17001	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17002
17003	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17004	{
17005	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17006	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17007	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
17008	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17009	AssertRCReturn(rc, rc);
17010
17011	VMXVEXITINFO ExitInfo;
17012	RT_ZERO(ExitInfo);
17013	ExitInfo.uReason = pVmxTransient->uExitReason;
17014	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17015	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17016	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17017	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17018	}
17019	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17020	}
17021
17022
17023	/**
17024	* Nested-guest VM-exit handler for invalid-guest state
17025	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17026	*/
17027	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17028	{
17029	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17030
17031	/*
17032	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17033	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17034	* Handle it like it's in an invalid guest state of the outer guest.
17035	*
17036	* When the fast path is implemented, this should be changed to cause the corresponding
17037	* nested-guest VM-exit.
17038	*/
17039	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17040	}
17041
17042
17043	/**
17044	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17045	* and only provide the instruction length.
17046	*
17047	* Unconditional VM-exit.
17048	*/
17049	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17050	{
17051	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17052
17053	#ifdef VBOX_STRICT
17054	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17055	switch (pVmxTransient->uExitReason)
17056	{
17057	case VMX_EXIT_ENCLS:
17058	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17059	break;
17060
17061	case VMX_EXIT_VMFUNC:
17062	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17063	break;
17064	}
17065	#endif
17066
17067	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17068	AssertRCReturn(rc, rc);
17069	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17070	}
17071
17072
17073	/**
17074	* Nested-guest VM-exit handler for instructions that provide instruction length as
17075	* well as more information.
17076	*
17077	* Unconditional VM-exit.
17078	*/
17079	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17080	{
17081	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17082
17083	#ifdef VBOX_STRICT
17084	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17085	switch (pVmxTransient->uExitReason)
17086	{
17087	case VMX_EXIT_GDTR_IDTR_ACCESS:
17088	case VMX_EXIT_LDTR_TR_ACCESS:
17089	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17090	break;
17091
17092	case VMX_EXIT_RDRAND:
17093	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17094	break;
17095
17096	case VMX_EXIT_RDSEED:
17097	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17098	break;
17099
17100	case VMX_EXIT_XSAVES:
17101	case VMX_EXIT_XRSTORS:
17102	/** @todo NSTVMX: Verify XSS-bitmap. */
17103	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17104	break;
17105
17106	case VMX_EXIT_UMWAIT:
17107	case VMX_EXIT_TPAUSE:
17108	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17109	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17110	break;
17111	}
17112	#endif
17113
17114	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17115	rc \|= hmR0VmxReadExitQualVmcs(pVmxTransient);
17116	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17117	AssertRCReturn(rc, rc);
17118
17119	VMXVEXITINFO ExitInfo;
17120	RT_ZERO(ExitInfo);
17121	ExitInfo.uReason = pVmxTransient->uExitReason;
17122	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17123	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17124	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17125	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17126	}
17127
17128	/** @} */
17129	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17130

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 80099

Download in other formats: