HMVMXR0.cpp@ 81002

Last change on this file since 81002 was 81002, checked in by vboxsync, 5 years ago
VMM: bugref:9566 TRPM enhancements and cleanup. Bumps TRPM saved-state version.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 710.0 KB

Line
1	/* $Id: HMVMXR0.cpp 81002 2019-09-25 09:12:34Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#ifdef VBOX_WITH_REM
39	# include <VBox/vmm/rem.h>
40	#endif
41	#include "HMInternal.h"
42	#include <VBox/vmm/vmcc.h>
43	#include <VBox/vmm/hmvmxinline.h>
44	#include "HMVMXR0.h"
45	#include "dtrace/VBoxVMM.h"
46
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
50	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
51	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
52	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
53	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
54	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
55	# define HMVMX_ALWAYS_TRAP_PF
56	# define HMVMX_ALWAYS_FLUSH_TLB
57	# define HMVMX_ALWAYS_SWAP_EFER
58	#endif
59
60
61	/*********************************************************************************************************************************
62	* Defined Constants And Macros *
63	*********************************************************************************************************************************/
64	/** Use the function table. */
65	#define HMVMX_USE_FUNCTION_TABLE
66
67	/** Determine which tagged-TLB flush handler to use. */
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
70	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
71	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
72
73	/**
74	* Flags to skip redundant reads of some common VMCS fields that are not part of
75	* the guest-CPU or VCPU state but are needed while handling VM-exits.
76	*/
77	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
78	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
79	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
80	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
81	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
82	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
83	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
84	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
85	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
86	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
87
88	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
89	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
90	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
91	\| HMVMX_READ_EXIT_INSTR_LEN \
92	\| HMVMX_READ_IDT_VECTORING_INFO \
93	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
94
95	/** Assert that all the given fields have been read from the VMCS. */
96	#ifdef VBOX_STRICT
97	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
98	do { \
99	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
100	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); \
101	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
102	} while (0)
103	#else
104	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
105	#endif
106
107	/**
108	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
109	* guest using hardware-assisted VMX.
110	*
111	* This excludes state like GPRs (other than RSP) which are always are
112	* swapped and restored across the world-switch and also registers like EFER,
113	* MSR which cannot be modified by the guest without causing a VM-exit.
114	*/
115	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
116	\| CPUMCTX_EXTRN_RFLAGS \
117	\| CPUMCTX_EXTRN_RSP \
118	\| CPUMCTX_EXTRN_SREG_MASK \
119	\| CPUMCTX_EXTRN_TABLE_MASK \
120	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
121	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
122	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
123	\| CPUMCTX_EXTRN_TSC_AUX \
124	\| CPUMCTX_EXTRN_OTHER_MSRS \
125	\| CPUMCTX_EXTRN_CR0 \
126	\| CPUMCTX_EXTRN_CR3 \
127	\| CPUMCTX_EXTRN_CR4 \
128	\| CPUMCTX_EXTRN_DR7 \
129	\| CPUMCTX_EXTRN_HWVIRT \
130	\| CPUMCTX_EXTRN_HM_VMX_MASK)
131
132	/**
133	* Exception bitmap mask for real-mode guests (real-on-v86).
134	*
135	* We need to intercept all exceptions manually except:
136	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
137	* due to bugs in Intel CPUs.
138	* - \#PF need not be intercepted even in real-mode if we have nested paging
139	* support.
140	*/
141	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
142	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
143	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
144	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
145	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
146	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
147	\| RT_BIT(X86_XCPT_XF))
148
149	/** Maximum VM-instruction error number. */
150	#define HMVMX_INSTR_ERROR_MAX 28
151
152	/** Profiling macro. */
153	#ifdef HM_PROFILE_EXIT_DISPATCH
154	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
156	#else
157	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
158	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
159	#endif
160
161	/** Assert that preemption is disabled or covered by thread-context hooks. */
162	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
163	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
164
165	/** Assert that we haven't migrated CPUs when thread-context hooks are not
166	* used. */
167	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
168	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
169	("Illegal migration! Entered on CPU %u Current %u\n", \
170	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
171
172	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
173	* context. */
174	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
175	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
176	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
177
178	/** Log the VM-exit reason with an easily visible marker to identify it in a
179	* potential sea of logging data. */
180	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
181	do { \
182	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
183	HMGetVmxExitName(a_uExitReason))); \
184	} while (0) \
185
186
187	/*********************************************************************************************************************************
188	* Structures and Typedefs *
189	*********************************************************************************************************************************/
190	/**
191	* VMX per-VCPU transient state.
192	*
193	* A state structure for holding miscellaneous information across
194	* VMX non-root operation and restored after the transition.
195	*/
196	typedef struct VMXTRANSIENT
197	{
198	/** The host's rflags/eflags. */
199	RTCCUINTREG fEFlags;
200
201	/** The guest's TPR value used for TPR shadowing. */
202	uint8_t u8GuestTpr;
203	/** Alignment. */
204	uint8_t abAlignment0[7];
205
206	/** The basic VM-exit reason. */
207	uint16_t uExitReason;
208	/** Alignment. */
209	uint16_t u16Alignment0;
210	/** The VM-exit interruption error code. */
211	uint32_t uExitIntErrorCode;
212	/** The VM-exit exit code qualification. */
213	uint64_t uExitQual;
214	/** The Guest-linear address. */
215	uint64_t uGuestLinearAddr;
216	/** The Guest-physical address. */
217	uint64_t uGuestPhysicalAddr;
218	/** The Guest pending-debug exceptions. */
219	uint64_t uGuestPendingDbgXcpts;
220
221	/** The VM-exit interruption-information field. */
222	uint32_t uExitIntInfo;
223	/** The VM-exit instruction-length field. */
224	uint32_t cbExitInstr;
225	/** The VM-exit instruction-information field. */
226	VMXEXITINSTRINFO ExitInstrInfo;
227	/** Whether the VM-entry failed or not. */
228	bool fVMEntryFailed;
229	/** Whether we are currently executing a nested-guest. */
230	bool fIsNestedGuest;
231	/** Alignment. */
232	uint8_t abAlignment1[2];
233
234	/** The VM-entry interruption-information field. */
235	uint32_t uEntryIntInfo;
236	/** The VM-entry exception error code field. */
237	uint32_t uEntryXcptErrorCode;
238	/** The VM-entry instruction length field. */
239	uint32_t cbEntryInstr;
240
241	/** IDT-vectoring information field. */
242	uint32_t uIdtVectoringInfo;
243	/** IDT-vectoring error code. */
244	uint32_t uIdtVectoringErrorCode;
245
246	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
247	uint32_t fVmcsFieldsRead;
248
249	/** Whether the guest debug state was active at the time of VM-exit. */
250	bool fWasGuestDebugStateActive;
251	/** Whether the hyper debug state was active at the time of VM-exit. */
252	bool fWasHyperDebugStateActive;
253	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
254	bool fUpdatedTscOffsettingAndPreemptTimer;
255	/** Whether the VM-exit was caused by a page-fault during delivery of a
256	* contributory exception or a page-fault. */
257	bool fVectoringDoublePF;
258	/** Whether the VM-exit was caused by a page-fault during delivery of an
259	* external interrupt or NMI. */
260	bool fVectoringPF;
261	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
262	* area after VM-exit. */
263	bool fRemoveTscAuxMsr;
264	bool afAlignment0[2];
265
266	/** The VMCS info. object. */
267	PVMXVMCSINFO pVmcsInfo;
268	} VMXTRANSIENT;
269	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
271	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
272	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
273	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
274	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
275	/** Pointer to VMX transient state. */
276	typedef VMXTRANSIENT *PVMXTRANSIENT;
277	/** Pointer to a const VMX transient state. */
278	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
279
280	/**
281	* Memory operand read or write access.
282	*/
283	typedef enum VMXMEMACCESS
284	{
285	VMXMEMACCESS_READ = 0,
286	VMXMEMACCESS_WRITE = 1
287	} VMXMEMACCESS;
288
289	/**
290	* VMX VM-exit handler.
291	*
292	* @returns Strict VBox status code (i.e. informational status codes too).
293	* @param pVCpu The cross context virtual CPU structure.
294	* @param pVmxTransient The VMX-transient structure.
295	*/
296	#ifndef HMVMX_USE_FUNCTION_TABLE
297	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
298	#else
299	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
300	/** Pointer to VM-exit handler. */
301	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
302	#endif
303
304	/**
305	* VMX VM-exit handler, non-strict status code.
306	*
307	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
308	*
309	* @returns VBox status code, no informational status code returned.
310	* @param pVCpu The cross context virtual CPU structure.
311	* @param pVmxTransient The VMX-transient structure.
312	*
313	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
314	* use of that status code will be replaced with VINF_EM_SOMETHING
315	* later when switching over to IEM.
316	*/
317	#ifndef HMVMX_USE_FUNCTION_TABLE
318	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
319	#else
320	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
321	#endif
322
323
324	/*********************************************************************************************************************************
325	* Internal Functions *
326	*********************************************************************************************************************************/
327	#ifndef HMVMX_USE_FUNCTION_TABLE
328	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
329	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
330	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
331	#else
332	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
333	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
334	#endif
335	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
336	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
337	#endif
338
339	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
340
341	/** @name VM-exit handler prototypes.
342	* @{
343	*/
344	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
345	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
346	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
347	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
348	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
349	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
350	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
351	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
352	static FNVMXEXITHANDLER hmR0VmxExitHlt;
353	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
354	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
355	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
356	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
357	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
358	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
359	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
360	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
361	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
362	static FNVMXEXITHANDLER hmR0VmxExitVmread;
363	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
364	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
365	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
366	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
367	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
368	#endif
369	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
370	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
371	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
372	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
373	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
374	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
375	static FNVMXEXITHANDLER hmR0VmxExitMwait;
376	static FNVMXEXITHANDLER hmR0VmxExitMtf;
377	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
378	static FNVMXEXITHANDLER hmR0VmxExitPause;
379	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
380	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
381	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
382	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
383	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
384	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
385	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
386	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
387	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
391	/** @} */
392
393	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
394	/** @name Nested-guest VM-exit handler prototypes.
395	* @{
396	*/
397	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
398	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
400	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
401	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
402	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
403	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
404	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
405	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
406	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
407	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
408	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
409	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
410	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
411	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
412	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
413	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
414	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
415	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
417	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
418	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
419	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
420	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
422	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
423	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
424	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
425	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
426	/** @} */
427	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
428
429
430	/*********************************************************************************************************************************
431	* Global Variables *
432	*********************************************************************************************************************************/
433	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
434	/**
435	* Array of all VMCS fields.
436	* Any fields added to the VT-x spec. should be added here.
437	*
438	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
439	* of nested-guests.
440	*/
441	static const uint32_t g_aVmcsFields[] =
442	{
443	/* 16-bit control fields. */
444	VMX_VMCS16_VPID,
445	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
446	VMX_VMCS16_EPTP_INDEX,
447
448	/* 16-bit guest-state fields. */
449	VMX_VMCS16_GUEST_ES_SEL,
450	VMX_VMCS16_GUEST_CS_SEL,
451	VMX_VMCS16_GUEST_SS_SEL,
452	VMX_VMCS16_GUEST_DS_SEL,
453	VMX_VMCS16_GUEST_FS_SEL,
454	VMX_VMCS16_GUEST_GS_SEL,
455	VMX_VMCS16_GUEST_LDTR_SEL,
456	VMX_VMCS16_GUEST_TR_SEL,
457	VMX_VMCS16_GUEST_INTR_STATUS,
458	VMX_VMCS16_GUEST_PML_INDEX,
459
460	/* 16-bits host-state fields. */
461	VMX_VMCS16_HOST_ES_SEL,
462	VMX_VMCS16_HOST_CS_SEL,
463	VMX_VMCS16_HOST_SS_SEL,
464	VMX_VMCS16_HOST_DS_SEL,
465	VMX_VMCS16_HOST_FS_SEL,
466	VMX_VMCS16_HOST_GS_SEL,
467	VMX_VMCS16_HOST_TR_SEL,
468
469	/* 64-bit control fields. */
470	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
471	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
472	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
473	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
474	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
475	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
476	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
477	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
478	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
479	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
480	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
481	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
482	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
483	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
484	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
485	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
486	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
487	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
488	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
489	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
490	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
491	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
492	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
493	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
494	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
495	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
496	VMX_VMCS64_CTRL_EPTP_FULL,
497	VMX_VMCS64_CTRL_EPTP_HIGH,
498	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
499	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
500	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
501	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
502	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
503	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
504	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
505	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
506	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
507	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
508	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
509	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
510	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
511	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
512	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
513	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
514	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
515	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
516	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
517	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
518	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
519	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
520
521	/* 64-bit read-only data fields. */
522	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
523	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
524
525	/* 64-bit guest-state fields. */
526	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
527	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
528	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
529	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
530	VMX_VMCS64_GUEST_PAT_FULL,
531	VMX_VMCS64_GUEST_PAT_HIGH,
532	VMX_VMCS64_GUEST_EFER_FULL,
533	VMX_VMCS64_GUEST_EFER_HIGH,
534	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
535	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
536	VMX_VMCS64_GUEST_PDPTE0_FULL,
537	VMX_VMCS64_GUEST_PDPTE0_HIGH,
538	VMX_VMCS64_GUEST_PDPTE1_FULL,
539	VMX_VMCS64_GUEST_PDPTE1_HIGH,
540	VMX_VMCS64_GUEST_PDPTE2_FULL,
541	VMX_VMCS64_GUEST_PDPTE2_HIGH,
542	VMX_VMCS64_GUEST_PDPTE3_FULL,
543	VMX_VMCS64_GUEST_PDPTE3_HIGH,
544	VMX_VMCS64_GUEST_BNDCFGS_FULL,
545	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
546
547	/* 64-bit host-state fields. */
548	VMX_VMCS64_HOST_PAT_FULL,
549	VMX_VMCS64_HOST_PAT_HIGH,
550	VMX_VMCS64_HOST_EFER_FULL,
551	VMX_VMCS64_HOST_EFER_HIGH,
552	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
553	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
554
555	/* 32-bit control fields. */
556	VMX_VMCS32_CTRL_PIN_EXEC,
557	VMX_VMCS32_CTRL_PROC_EXEC,
558	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
559	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
560	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
561	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
562	VMX_VMCS32_CTRL_EXIT,
563	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
564	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
565	VMX_VMCS32_CTRL_ENTRY,
566	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
567	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
568	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
569	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
570	VMX_VMCS32_CTRL_TPR_THRESHOLD,
571	VMX_VMCS32_CTRL_PROC_EXEC2,
572	VMX_VMCS32_CTRL_PLE_GAP,
573	VMX_VMCS32_CTRL_PLE_WINDOW,
574
575	/* 32-bits read-only fields. */
576	VMX_VMCS32_RO_VM_INSTR_ERROR,
577	VMX_VMCS32_RO_EXIT_REASON,
578	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
579	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
580	VMX_VMCS32_RO_IDT_VECTORING_INFO,
581	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
582	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
583	VMX_VMCS32_RO_EXIT_INSTR_INFO,
584
585	/* 32-bit guest-state fields. */
586	VMX_VMCS32_GUEST_ES_LIMIT,
587	VMX_VMCS32_GUEST_CS_LIMIT,
588	VMX_VMCS32_GUEST_SS_LIMIT,
589	VMX_VMCS32_GUEST_DS_LIMIT,
590	VMX_VMCS32_GUEST_FS_LIMIT,
591	VMX_VMCS32_GUEST_GS_LIMIT,
592	VMX_VMCS32_GUEST_LDTR_LIMIT,
593	VMX_VMCS32_GUEST_TR_LIMIT,
594	VMX_VMCS32_GUEST_GDTR_LIMIT,
595	VMX_VMCS32_GUEST_IDTR_LIMIT,
596	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
597	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
598	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
599	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
600	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
602	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
603	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
604	VMX_VMCS32_GUEST_INT_STATE,
605	VMX_VMCS32_GUEST_ACTIVITY_STATE,
606	VMX_VMCS32_GUEST_SMBASE,
607	VMX_VMCS32_GUEST_SYSENTER_CS,
608	VMX_VMCS32_PREEMPT_TIMER_VALUE,
609
610	/* 32-bit host-state fields. */
611	VMX_VMCS32_HOST_SYSENTER_CS,
612
613	/* Natural-width control fields. */
614	VMX_VMCS_CTRL_CR0_MASK,
615	VMX_VMCS_CTRL_CR4_MASK,
616	VMX_VMCS_CTRL_CR0_READ_SHADOW,
617	VMX_VMCS_CTRL_CR4_READ_SHADOW,
618	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
619	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
620	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
621	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
622
623	/* Natural-width read-only data fields. */
624	VMX_VMCS_RO_EXIT_QUALIFICATION,
625	VMX_VMCS_RO_IO_RCX,
626	VMX_VMCS_RO_IO_RSI,
627	VMX_VMCS_RO_IO_RDI,
628	VMX_VMCS_RO_IO_RIP,
629	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
630
631	/* Natural-width guest-state field */
632	VMX_VMCS_GUEST_CR0,
633	VMX_VMCS_GUEST_CR3,
634	VMX_VMCS_GUEST_CR4,
635	VMX_VMCS_GUEST_ES_BASE,
636	VMX_VMCS_GUEST_CS_BASE,
637	VMX_VMCS_GUEST_SS_BASE,
638	VMX_VMCS_GUEST_DS_BASE,
639	VMX_VMCS_GUEST_FS_BASE,
640	VMX_VMCS_GUEST_GS_BASE,
641	VMX_VMCS_GUEST_LDTR_BASE,
642	VMX_VMCS_GUEST_TR_BASE,
643	VMX_VMCS_GUEST_GDTR_BASE,
644	VMX_VMCS_GUEST_IDTR_BASE,
645	VMX_VMCS_GUEST_DR7,
646	VMX_VMCS_GUEST_RSP,
647	VMX_VMCS_GUEST_RIP,
648	VMX_VMCS_GUEST_RFLAGS,
649	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
650	VMX_VMCS_GUEST_SYSENTER_ESP,
651	VMX_VMCS_GUEST_SYSENTER_EIP,
652
653	/* Natural-width host-state fields */
654	VMX_VMCS_HOST_CR0,
655	VMX_VMCS_HOST_CR3,
656	VMX_VMCS_HOST_CR4,
657	VMX_VMCS_HOST_FS_BASE,
658	VMX_VMCS_HOST_GS_BASE,
659	VMX_VMCS_HOST_TR_BASE,
660	VMX_VMCS_HOST_GDTR_BASE,
661	VMX_VMCS_HOST_IDTR_BASE,
662	VMX_VMCS_HOST_SYSENTER_ESP,
663	VMX_VMCS_HOST_SYSENTER_EIP,
664	VMX_VMCS_HOST_RSP,
665	VMX_VMCS_HOST_RIP
666	};
667	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
668
669	static const uint32_t g_aVmcsSegBase[] =
670	{
671	VMX_VMCS_GUEST_ES_BASE,
672	VMX_VMCS_GUEST_CS_BASE,
673	VMX_VMCS_GUEST_SS_BASE,
674	VMX_VMCS_GUEST_DS_BASE,
675	VMX_VMCS_GUEST_FS_BASE,
676	VMX_VMCS_GUEST_GS_BASE
677	};
678	static const uint32_t g_aVmcsSegSel[] =
679	{
680	VMX_VMCS16_GUEST_ES_SEL,
681	VMX_VMCS16_GUEST_CS_SEL,
682	VMX_VMCS16_GUEST_SS_SEL,
683	VMX_VMCS16_GUEST_DS_SEL,
684	VMX_VMCS16_GUEST_FS_SEL,
685	VMX_VMCS16_GUEST_GS_SEL
686	};
687	static const uint32_t g_aVmcsSegLimit[] =
688	{
689	VMX_VMCS32_GUEST_ES_LIMIT,
690	VMX_VMCS32_GUEST_CS_LIMIT,
691	VMX_VMCS32_GUEST_SS_LIMIT,
692	VMX_VMCS32_GUEST_DS_LIMIT,
693	VMX_VMCS32_GUEST_FS_LIMIT,
694	VMX_VMCS32_GUEST_GS_LIMIT
695	};
696	static const uint32_t g_aVmcsSegAttr[] =
697	{
698	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
699	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
700	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
701	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
702	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
703	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
704	};
705	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
706	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
707	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
708	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
709
710	#ifdef HMVMX_USE_FUNCTION_TABLE
711	/**
712	* VMX_EXIT dispatch table.
713	*/
714	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
715	{
716	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
717	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
718	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
719	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
720	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
721	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
722	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
723	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
724	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
725	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
726	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
727	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
728	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
729	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
730	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
731	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
732	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
733	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
734	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
735	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
736	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
737	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
738	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
739	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
740	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
741	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
742	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
743	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
744	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
745	#else
746	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
747	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
748	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
749	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
750	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
751	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
752	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
753	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
754	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
755	#endif
756	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
757	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
758	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
759	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
760	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
761	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
762	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
763	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
764	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
765	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
766	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
767	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
768	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
769	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
770	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
771	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
772	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
773	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
774	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
775	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
776	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
777	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
778	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
779	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
780	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
781	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
782	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
783	#else
784	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
785	#endif
786	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
787	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
788	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
789	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
790	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
791	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
792	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
793	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
794	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
795	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
796	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
797	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
798	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
799	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
800	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
801	};
802	#endif /* HMVMX_USE_FUNCTION_TABLE */
803
804	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
805	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
806	{
807	/* 0 */ "(Not Used)",
808	/* 1 */ "VMCALL executed in VMX root operation.",
809	/* 2 */ "VMCLEAR with invalid physical address.",
810	/* 3 */ "VMCLEAR with VMXON pointer.",
811	/* 4 */ "VMLAUNCH with non-clear VMCS.",
812	/* 5 */ "VMRESUME with non-launched VMCS.",
813	/* 6 */ "VMRESUME after VMXOFF",
814	/* 7 */ "VM-entry with invalid control fields.",
815	/* 8 */ "VM-entry with invalid host state fields.",
816	/* 9 */ "VMPTRLD with invalid physical address.",
817	/* 10 */ "VMPTRLD with VMXON pointer.",
818	/* 11 */ "VMPTRLD with incorrect revision identifier.",
819	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
820	/* 13 */ "VMWRITE to read-only VMCS component.",
821	/* 14 */ "(Not Used)",
822	/* 15 */ "VMXON executed in VMX root operation.",
823	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
824	/* 17 */ "VM-entry with non-launched executing VMCS.",
825	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
826	/* 19 */ "VMCALL with non-clear VMCS.",
827	/* 20 */ "VMCALL with invalid VM-exit control fields.",
828	/* 21 */ "(Not Used)",
829	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
830	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
831	/* 24 */ "VMCALL with invalid SMM-monitor features.",
832	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
833	/* 26 */ "VM-entry with events blocked by MOV SS.",
834	/* 27 */ "(Not Used)",
835	/* 28 */ "Invalid operand to INVEPT/INVVPID."
836	};
837	#endif /* VBOX_STRICT && LOG_ENABLED */
838
839
840	/**
841	* Gets the CR0 guest/host mask.
842	*
843	* These bits typically does not change through the lifetime of a VM. Any bit set in
844	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
845	* by the guest.
846	*
847	* @returns The CR0 guest/host mask.
848	* @param pVCpu The cross context virtual CPU structure.
849	*/
850	static uint64_t hmR0VmxGetFixedCr0Mask(PCVMCPUCC pVCpu)
851	{
852	/*
853	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
854	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
855	*
856	* Furthermore, modifications to any bits that are reserved/unspecified currently
857	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
858	* when future CPUs specify and use currently reserved/unspecified bits.
859	*/
860	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
861	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
862	* and @bugref{6944}. */
863	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
864	return ( X86_CR0_PE
865	\| X86_CR0_NE
866	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
867	\| X86_CR0_PG
868	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
869	}
870
871
872	/**
873	* Gets the CR4 guest/host mask.
874	*
875	* These bits typically does not change through the lifetime of a VM. Any bit set in
876	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
877	* by the guest.
878	*
879	* @returns The CR4 guest/host mask.
880	* @param pVCpu The cross context virtual CPU structure.
881	*/
882	static uint64_t hmR0VmxGetFixedCr4Mask(PCVMCPUCC pVCpu)
883	{
884	/*
885	* We construct a mask of all CR4 bits that the guest can modify without causing
886	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
887	* a VM-exit when the guest attempts to modify them when executing using
888	* hardware-assisted VMX.
889	*
890	* When a feature is not exposed to the guest (and may be present on the host),
891	* we want to intercept guest modifications to the bit so we can emulate proper
892	* behavior (e.g., #GP).
893	*
894	* Furthermore, only modifications to those bits that don't require immediate
895	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
896	* depends on CR3 which might not always be the guest value while executing
897	* using hardware-assisted VMX.
898	*/
899	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
900	bool const fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
901	bool const fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
902	bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
903
904	/*
905	* Paranoia.
906	* Ensure features exposed to the guest are present on the host.
907	*/
908	Assert(!fFsGsBase \|\| pVM->cpum.ro.HostFeatures.fFsGsBase);
909	Assert(!fXSaveRstor \|\| pVM->cpum.ro.HostFeatures.fXSaveRstor);
910	Assert(!fFxSaveRstor \|\| pVM->cpum.ro.HostFeatures.fFxSaveRstor);
911
912	uint64_t const fGstMask = ( X86_CR4_PVI
913	\| X86_CR4_TSD
914	\| X86_CR4_DE
915	\| X86_CR4_MCE
916	\| X86_CR4_PCE
917	\| X86_CR4_OSXMMEEXCPT
918	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
919	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
920	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0));
921	return ~fGstMask;
922	}
923
924
925	/**
926	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
927	* area.
928	*
929	* @returns @c true if it's different, @c false otherwise.
930	* @param pVmcsInfo The VMCS info. object.
931	*/
932	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
933	{
934	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
935	&& pVmcsInfo->pvGuestMsrStore);
936	}
937
938
939	/**
940	* Sets the given Processor-based VM-execution controls.
941	*
942	* @param pVmxTransient The VMX-transient structure.
943	* @param uProcCtls The Processor-based VM-execution controls to set.
944	*/
945	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
946	{
947	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
948	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
949	{
950	pVmcsInfo->u32ProcCtls \|= uProcCtls;
951	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
952	AssertRC(rc);
953	}
954	}
955
956
957	/**
958	* Removes the given Processor-based VM-execution controls.
959	*
960	* @param pVCpu The cross context virtual CPU structure.
961	* @param pVmxTransient The VMX-transient structure.
962	* @param uProcCtls The Processor-based VM-execution controls to remove.
963	*
964	* @remarks When executing a nested-guest, this will not remove any of the specified
965	* controls if the nested hypervisor has set any one of them.
966	*/
967	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
968	{
969	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
970	if (pVmcsInfo->u32ProcCtls & uProcCtls)
971	{
972	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
973	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
974	? true
975	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
976	#else
977	NOREF(pVCpu);
978	bool const fRemoveCtls = true;
979	#endif
980	if (fRemoveCtls)
981	{
982	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
983	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
984	AssertRC(rc);
985	}
986	}
987	}
988
989
990	/**
991	* Sets the TSC offset for the current VMCS.
992	*
993	* @param uTscOffset The TSC offset to set.
994	* @param pVmcsInfo The VMCS info. object.
995	*/
996	static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
997	{
998	if (pVmcsInfo->u64TscOffset != uTscOffset)
999	{
1000	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
1001	AssertRC(rc);
1002	pVmcsInfo->u64TscOffset = uTscOffset;
1003	}
1004	}
1005
1006
1007	/**
1008	* Adds one or more exceptions to the exception bitmap and commits it to the current
1009	* VMCS.
1010	*
1011	* @param pVmxTransient The VMX-transient structure.
1012	* @param uXcptMask The exception(s) to add.
1013	*/
1014	static void hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1015	{
1016	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1017	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1018	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1019	{
1020	uXcptBitmap \|= uXcptMask;
1021	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1022	AssertRC(rc);
1023	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1024	}
1025	}
1026
1027
1028	/**
1029	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1030	*
1031	* @param pVmxTransient The VMX-transient structure.
1032	* @param uXcpt The exception to add.
1033	*/
1034	static void hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1035	{
1036	Assert(uXcpt <= X86_XCPT_LAST);
1037	hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1038	}
1039
1040
1041	/**
1042	* Remove one or more exceptions from the exception bitmap and commits it to the
1043	* current VMCS.
1044	*
1045	* This takes care of not removing the exception intercept if a nested-guest
1046	* requires the exception to be intercepted.
1047	*
1048	* @returns VBox status code.
1049	* @param pVCpu The cross context virtual CPU structure.
1050	* @param pVmxTransient The VMX-transient structure.
1051	* @param uXcptMask The exception(s) to remove.
1052	*/
1053	static int hmR0VmxRemoveXcptInterceptMask(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1054	{
1055	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1056	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1057	if (u32XcptBitmap & uXcptMask)
1058	{
1059	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1060	if (!pVmxTransient->fIsNestedGuest)
1061	{ /* likely */ }
1062	else
1063	{
1064	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1065	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1066	}
1067	#endif
1068	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1069	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1070	\| RT_BIT(X86_XCPT_DE)
1071	\| RT_BIT(X86_XCPT_NM)
1072	\| RT_BIT(X86_XCPT_TS)
1073	\| RT_BIT(X86_XCPT_UD)
1074	\| RT_BIT(X86_XCPT_NP)
1075	\| RT_BIT(X86_XCPT_SS)
1076	\| RT_BIT(X86_XCPT_GP)
1077	\| RT_BIT(X86_XCPT_PF)
1078	\| RT_BIT(X86_XCPT_MF));
1079	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1080	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1081	#endif
1082	if (uXcptMask)
1083	{
1084	/* Validate we are not removing any essential exception intercepts. */
1085	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1086	NOREF(pVCpu);
1087	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1088	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1089
1090	/* Remove it from the exception bitmap. */
1091	u32XcptBitmap &= ~uXcptMask;
1092
1093	/* Commit and update the cache if necessary. */
1094	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1095	{
1096	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1097	AssertRC(rc);
1098	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1099	}
1100	}
1101	}
1102	return VINF_SUCCESS;
1103	}
1104
1105
1106	/**
1107	* Remove an exceptions from the exception bitmap and commits it to the current
1108	* VMCS.
1109	*
1110	* @returns VBox status code.
1111	* @param pVCpu The cross context virtual CPU structure.
1112	* @param pVmxTransient The VMX-transient structure.
1113	* @param uXcpt The exception to remove.
1114	*/
1115	static int hmR0VmxRemoveXcptIntercept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1116	{
1117	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1118	}
1119
1120
1121	/**
1122	* Loads the VMCS specified by the VMCS info. object.
1123	*
1124	* @returns VBox status code.
1125	* @param pVmcsInfo The VMCS info. object.
1126	*
1127	* @remarks Can be called with interrupts disabled.
1128	*/
1129	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1130	{
1131	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1132	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1133
1134	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1135	if (RT_SUCCESS(rc))
1136	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1137	return rc;
1138	}
1139
1140
1141	/**
1142	* Clears the VMCS specified by the VMCS info. object.
1143	*
1144	* @returns VBox status code.
1145	* @param pVmcsInfo The VMCS info. object.
1146	*
1147	* @remarks Can be called with interrupts disabled.
1148	*/
1149	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1150	{
1151	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1152	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1153
1154	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1155	if (RT_SUCCESS(rc))
1156	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1157	return rc;
1158	}
1159
1160
1161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1162	/**
1163	* Loads the shadow VMCS specified by the VMCS info. object.
1164	*
1165	* @returns VBox status code.
1166	* @param pVmcsInfo The VMCS info. object.
1167	*
1168	* @remarks Can be called with interrupts disabled.
1169	*/
1170	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1171	{
1172	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1173	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1174
1175	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1176	if (RT_SUCCESS(rc))
1177	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1178	return rc;
1179	}
1180
1181
1182	/**
1183	* Clears the shadow VMCS specified by the VMCS info. object.
1184	*
1185	* @returns VBox status code.
1186	* @param pVmcsInfo The VMCS info. object.
1187	*
1188	* @remarks Can be called with interrupts disabled.
1189	*/
1190	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1191	{
1192	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1193	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1194
1195	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1196	if (RT_SUCCESS(rc))
1197	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1198	return rc;
1199	}
1200
1201
1202	/**
1203	* Switches from and to the specified VMCSes.
1204	*
1205	* @returns VBox status code.
1206	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1207	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1208	*
1209	* @remarks Called with interrupts disabled.
1210	*/
1211	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1212	{
1213	/*
1214	* Clear the VMCS we are switching out if it has not already been cleared.
1215	* This will sync any CPU internal data back to the VMCS.
1216	*/
1217	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1218	{
1219	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1220	if (RT_SUCCESS(rc))
1221	{
1222	/*
1223	* The shadow VMCS, if any, would not be active at this point since we
1224	* would have cleared it while importing the virtual hardware-virtualization
1225	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1226	* clear the shadow VMCS here, just assert for safety.
1227	*/
1228	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1229	}
1230	else
1231	return rc;
1232	}
1233
1234	/*
1235	* Clear the VMCS we are switching to if it has not already been cleared.
1236	* This will initialize the VMCS launch state to "clear" required for loading it.
1237	*
1238	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1239	*/
1240	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1241	{
1242	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1243	if (RT_SUCCESS(rc))
1244	{ /* likely */ }
1245	else
1246	return rc;
1247	}
1248
1249	/*
1250	* Finally, load the VMCS we are switching to.
1251	*/
1252	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1253	}
1254
1255
1256	/**
1257	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1258	* caller.
1259	*
1260	* @returns VBox status code.
1261	* @param pVCpu The cross context virtual CPU structure.
1262	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1263	* true) or guest VMCS (pass false).
1264	*/
1265	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1266	{
1267	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1268	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1269
1270	PVMXVMCSINFO pVmcsInfoFrom;
1271	PVMXVMCSINFO pVmcsInfoTo;
1272	if (fSwitchToNstGstVmcs)
1273	{
1274	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1275	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1276	}
1277	else
1278	{
1279	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1280	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1281	}
1282
1283	/*
1284	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1285	* preemption hook code path acquires the current VMCS.
1286	*/
1287	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1288
1289	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1290	if (RT_SUCCESS(rc))
1291	{
1292	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1293
1294	/*
1295	* If we are switching to a VMCS that was executed on a different host CPU or was
1296	* never executed before, flag that we need to export the host state before executing
1297	* guest/nested-guest code using hardware-assisted VMX.
1298	*
1299	* This could probably be done in a preemptible context since the preemption hook
1300	* will flag the necessary change in host context. However, since preemption is
1301	* already disabled and to avoid making assumptions about host specific code in
1302	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1303	* disabled.
1304	*/
1305	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1306	{ /* likely */ }
1307	else
1308	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1309
1310	ASMSetFlags(fEFlags);
1311
1312	/*
1313	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1314	* flag that we need to update the host MSR values there. Even if we decide in the
1315	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1316	* if its content differs, we would have to update the host MSRs anyway.
1317	*/
1318	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1319	}
1320	else
1321	ASMSetFlags(fEFlags);
1322	return rc;
1323	}
1324	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1325
1326
1327	/**
1328	* Updates the VM's last error record.
1329	*
1330	* If there was a VMX instruction error, reads the error data from the VMCS and
1331	* updates VCPU's last error record as well.
1332	*
1333	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1334	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1335	* VERR_VMX_INVALID_VMCS_FIELD.
1336	* @param rc The error code.
1337	*/
1338	static void hmR0VmxUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
1339	{
1340	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1341	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1342	{
1343	AssertPtrReturnVoid(pVCpu);
1344	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1345	}
1346	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1347	}
1348
1349
1350	#ifdef VBOX_STRICT
1351	/**
1352	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1353	* transient structure.
1354	*
1355	* @param pVmxTransient The VMX-transient structure.
1356	*/
1357	DECLINLINE(void) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1358	{
1359	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1360	AssertRC(rc);
1361	}
1362
1363
1364	/**
1365	* Reads the VM-entry exception error code field from the VMCS into
1366	* the VMX transient structure.
1367	*
1368	* @param pVmxTransient The VMX-transient structure.
1369	*/
1370	DECLINLINE(void) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1371	{
1372	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1373	AssertRC(rc);
1374	}
1375
1376
1377	/**
1378	* Reads the VM-entry exception error code field from the VMCS into
1379	* the VMX transient structure.
1380	*
1381	* @param pVmxTransient The VMX-transient structure.
1382	*/
1383	DECLINLINE(void) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1384	{
1385	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1386	AssertRC(rc);
1387	}
1388	#endif /* VBOX_STRICT */
1389
1390
1391	/**
1392	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1393	* transient structure.
1394	*
1395	* @param pVmxTransient The VMX-transient structure.
1396	*/
1397	DECLINLINE(void) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1398	{
1399	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1400	{
1401	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1402	AssertRC(rc);
1403	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1404	}
1405	}
1406
1407
1408	/**
1409	* Reads the VM-exit interruption error code from the VMCS into the VMX
1410	* transient structure.
1411	*
1412	* @param pVmxTransient The VMX-transient structure.
1413	*/
1414	DECLINLINE(void) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1415	{
1416	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1417	{
1418	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1419	AssertRC(rc);
1420	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1421	}
1422	}
1423
1424
1425	/**
1426	* Reads the VM-exit instruction length field from the VMCS into the VMX
1427	* transient structure.
1428	*
1429	* @param pVmxTransient The VMX-transient structure.
1430	*/
1431	DECLINLINE(void) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1432	{
1433	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1434	{
1435	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1436	AssertRC(rc);
1437	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1438	}
1439	}
1440
1441
1442	/**
1443	* Reads the VM-exit instruction-information field from the VMCS into
1444	* the VMX transient structure.
1445	*
1446	* @param pVmxTransient The VMX-transient structure.
1447	*/
1448	DECLINLINE(void) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1449	{
1450	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1451	{
1452	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1453	AssertRC(rc);
1454	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1455	}
1456	}
1457
1458
1459	/**
1460	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1461	*
1462	* @param pVmxTransient The VMX-transient structure.
1463	*/
1464	DECLINLINE(void) hmR0VmxReadExitQualVmcs(PVMXTRANSIENT pVmxTransient)
1465	{
1466	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1467	{
1468	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1469	AssertRC(rc);
1470	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1471	}
1472	}
1473
1474
1475	/**
1476	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1477	*
1478	* @param pVmxTransient The VMX-transient structure.
1479	*/
1480	DECLINLINE(void) hmR0VmxReadGuestLinearAddrVmcs(PVMXTRANSIENT pVmxTransient)
1481	{
1482	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1483	{
1484	int rc = VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1485	AssertRC(rc);
1486	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1487	}
1488	}
1489
1490
1491	/**
1492	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1493	*
1494	* @param pVmxTransient The VMX-transient structure.
1495	*/
1496	DECLINLINE(void) hmR0VmxReadGuestPhysicalAddrVmcs(PVMXTRANSIENT pVmxTransient)
1497	{
1498	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1499	{
1500	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1501	AssertRC(rc);
1502	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1503	}
1504	}
1505
1506	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1507	/**
1508	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1509	* structure.
1510	*
1511	* @param pVmxTransient The VMX-transient structure.
1512	*/
1513	DECLINLINE(void) hmR0VmxReadGuestPendingDbgXctps(PVMXTRANSIENT pVmxTransient)
1514	{
1515	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1516	{
1517	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1518	AssertRC(rc);
1519	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1520	}
1521	}
1522	#endif
1523
1524	/**
1525	* Reads the IDT-vectoring information field from the VMCS into the VMX
1526	* transient structure.
1527	*
1528	* @param pVmxTransient The VMX-transient structure.
1529	*
1530	* @remarks No-long-jump zone!!!
1531	*/
1532	DECLINLINE(void) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1533	{
1534	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1535	{
1536	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1537	AssertRC(rc);
1538	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1539	}
1540	}
1541
1542
1543	/**
1544	* Reads the IDT-vectoring error code from the VMCS into the VMX
1545	* transient structure.
1546	*
1547	* @param pVmxTransient The VMX-transient structure.
1548	*/
1549	DECLINLINE(void) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1550	{
1551	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1552	{
1553	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1554	AssertRC(rc);
1555	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1556	}
1557	}
1558
1559	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1560	/**
1561	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1562	*
1563	* @param pVmxTransient The VMX-transient structure.
1564	*/
1565	static void hmR0VmxReadAllRoFieldsVmcs(PVMXTRANSIENT pVmxTransient)
1566	{
1567	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1568	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1569	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1570	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1571	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1572	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1573	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1574	rc \|= VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1575	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1576	AssertRC(rc);
1577	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1578	\| HMVMX_READ_EXIT_INSTR_LEN
1579	\| HMVMX_READ_EXIT_INSTR_INFO
1580	\| HMVMX_READ_IDT_VECTORING_INFO
1581	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1582	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1583	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1584	\| HMVMX_READ_GUEST_LINEAR_ADDR
1585	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1586	}
1587	#endif
1588
1589	/**
1590	* Enters VMX root mode operation on the current CPU.
1591	*
1592	* @returns VBox status code.
1593	* @param pHostCpu The HM physical-CPU structure.
1594	* @param pVM The cross context VM structure. Can be
1595	* NULL, after a resume.
1596	* @param HCPhysCpuPage Physical address of the VMXON region.
1597	* @param pvCpuPage Pointer to the VMXON region.
1598	*/
1599	static int hmR0VmxEnterRootMode(PHMPHYSCPU pHostCpu, PVMCC pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1600	{
1601	Assert(pHostCpu);
1602	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1603	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1604	Assert(pvCpuPage);
1605	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1606
1607	if (pVM)
1608	{
1609	/* Write the VMCS revision identifier to the VMXON region. */
1610	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1611	}
1612
1613	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1614	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1615
1616	/* Enable the VMX bit in CR4 if necessary. */
1617	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1618
1619	/* Record whether VMXE was already prior to us enabling it above. */
1620	pHostCpu->fVmxeAlreadyEnabled = RT_BOOL(uOldCr4 & X86_CR4_VMXE);
1621
1622	/* Enter VMX root mode. */
1623	int rc = VMXEnable(HCPhysCpuPage);
1624	if (RT_FAILURE(rc))
1625	{
1626	/* Restore CR4.VMXE if it was not set prior to our attempt to set it above. */
1627	if (!pHostCpu->fVmxeAlreadyEnabled)
1628	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1629
1630	if (pVM)
1631	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1632	}
1633
1634	/* Restore interrupts. */
1635	ASMSetFlags(fEFlags);
1636	return rc;
1637	}
1638
1639
1640	/**
1641	* Exits VMX root mode operation on the current CPU.
1642	*
1643	* @returns VBox status code.
1644	* @param pHostCpu The HM physical-CPU structure.
1645	*/
1646	static int hmR0VmxLeaveRootMode(PHMPHYSCPU pHostCpu)
1647	{
1648	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1649
1650	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1651	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1652
1653	/* If we're for some reason not in VMX root mode, then don't leave it. */
1654	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1655
1656	int rc;
1657	if (uHostCr4 & X86_CR4_VMXE)
1658	{
1659	/* Exit VMX root mode and clear the VMX bit in CR4. */
1660	VMXDisable();
1661
1662	/* Clear CR4.VMXE only if it was clear prior to use setting it. */
1663	if (!pHostCpu->fVmxeAlreadyEnabled)
1664	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
1665
1666	rc = VINF_SUCCESS;
1667	}
1668	else
1669	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1670
1671	/* Restore interrupts. */
1672	ASMSetFlags(fEFlags);
1673	return rc;
1674	}
1675
1676
1677	/**
1678	* Allocates and maps a physically contiguous page. The allocated page is
1679	* zero'd out (used by various VT-x structures).
1680	*
1681	* @returns IPRT status code.
1682	* @param pMemObj Pointer to the ring-0 memory object.
1683	* @param ppVirt Where to store the virtual address of the allocation.
1684	* @param pHCPhys Where to store the physical address of the allocation.
1685	*/
1686	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1687	{
1688	AssertPtr(pMemObj);
1689	AssertPtr(ppVirt);
1690	AssertPtr(pHCPhys);
1691	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1692	if (RT_FAILURE(rc))
1693	return rc;
1694	ppVirt = RTR0MemObjAddress(pMemObj);
1695	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1696	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1697	return VINF_SUCCESS;
1698	}
1699
1700
1701	/**
1702	* Frees and unmaps an allocated, physical page.
1703	*
1704	* @param pMemObj Pointer to the ring-0 memory object.
1705	* @param ppVirt Where to re-initialize the virtual address of allocation as
1706	* 0.
1707	* @param pHCPhys Where to re-initialize the physical address of the
1708	* allocation as 0.
1709	*/
1710	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1711	{
1712	AssertPtr(pMemObj);
1713	AssertPtr(ppVirt);
1714	AssertPtr(pHCPhys);
1715	/* NULL is valid, accepted and ignored by the free function below. */
1716	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1717	*pMemObj = NIL_RTR0MEMOBJ;
1718	*ppVirt = NULL;
1719	*pHCPhys = NIL_RTHCPHYS;
1720	}
1721
1722
1723	/**
1724	* Initializes a VMCS info. object.
1725	*
1726	* @param pVmcsInfo The VMCS info. object.
1727	*/
1728	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1729	{
1730	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1731
1732	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1733	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1734	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1735	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1736	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1737	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1738	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1739	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1740	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1741	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1742	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1743	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1744	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1745	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1746	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1747	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
1748	pVmcsInfo->idHostCpuExec = NIL_RTCPUID;
1749	}
1750
1751
1752	/**
1753	* Frees the VT-x structures for a VMCS info. object.
1754	*
1755	* @param pVM The cross context VM structure.
1756	* @param pVmcsInfo The VMCS info. object.
1757	*/
1758	static void hmR0VmxFreeVmcsInfo(PVMCC pVM, PVMXVMCSINFO pVmcsInfo)
1759	{
1760	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1761
1762	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1763	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1764	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1765	#endif
1766
1767	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1768	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1769
1770	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1771	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1772	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1773
1774	hmR0VmxInitVmcsInfo(pVmcsInfo);
1775	}
1776
1777
1778	/**
1779	* Allocates the VT-x structures for a VMCS info. object.
1780	*
1781	* @returns VBox status code.
1782	* @param pVCpu The cross context virtual CPU structure.
1783	* @param pVmcsInfo The VMCS info. object.
1784	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1785	*/
1786	static int hmR0VmxAllocVmcsInfo(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1787	{
1788	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1789
1790	/* Allocate the guest VM control structure (VMCS). */
1791	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1792	if (RT_SUCCESS(rc))
1793	{
1794	if (!fIsNstGstVmcs)
1795	{
1796	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1797	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1798	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1799	#endif
1800	if (RT_SUCCESS(rc))
1801	{
1802	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1803	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1804	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1805	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1806	}
1807	}
1808	else
1809	{
1810	/* We don't yet support exposing VMCS shadowing to the guest. */
1811	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1812	Assert(!pVmcsInfo->pvShadowVmcs);
1813
1814	/* Get the allocated virtual-APIC page from CPUM. */
1815	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1816	{
1817	/** @todo NSTVMX: Get rid of this. There is no need to allocate a separate HC
1818	* page for this. Use the one provided by the nested-guest directly. */
1819	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1820	&pVmcsInfo->HCPhysVirtApic);
1821	Assert(pVmcsInfo->pbVirtApic);
1822	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1823	}
1824	}
1825
1826	if (RT_SUCCESS(rc))
1827	{
1828	/*
1829	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1830	* transparent accesses of specific MSRs.
1831	*
1832	* If the condition for enabling MSR bitmaps changes here, don't forget to
1833	* update HMIsMsrBitmapActive().
1834	*
1835	* We don't share MSR bitmaps between the guest and nested-guest as we then
1836	* don't need to care about carefully restoring the guest MSR bitmap.
1837	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1838	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1839	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1840	* we do that later while merging VMCS.
1841	*/
1842	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1843	{
1844	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1845	if ( RT_SUCCESS(rc)
1846	&& !fIsNstGstVmcs)
1847	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1848	}
1849
1850	if (RT_SUCCESS(rc))
1851	{
1852	/*
1853	* Allocate the VM-entry MSR-load area for the guest MSRs.
1854	*
1855	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1856	* the guest and nested-guest.
1857	*/
1858	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1859	&pVmcsInfo->HCPhysGuestMsrLoad);
1860	if (RT_SUCCESS(rc))
1861	{
1862	/*
1863	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1864	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1865	*/
1866	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1867	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1868	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1869
1870	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1871	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1872	&pVmcsInfo->HCPhysHostMsrLoad);
1873	}
1874	}
1875	}
1876	}
1877
1878	return rc;
1879	}
1880
1881
1882	/**
1883	* Free all VT-x structures for the VM.
1884	*
1885	* @returns IPRT status code.
1886	* @param pVM The cross context VM structure.
1887	*/
1888	static void hmR0VmxStructsFree(PVMCC pVM)
1889	{
1890	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1891	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1892	#endif
1893	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1894
1895	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1896	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1897	{
1898	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1899	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1900	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1901	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1902	}
1903	#endif
1904
1905	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1906	{
1907	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1908	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1909	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1910	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1911	if (pVM->cpum.ro.GuestFeatures.fVmx)
1912	{
1913	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1914	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1915	}
1916	#endif
1917	}
1918	}
1919
1920
1921	/**
1922	* Allocate all VT-x structures for the VM.
1923	*
1924	* @returns IPRT status code.
1925	* @param pVM The cross context VM structure.
1926	*/
1927	static int hmR0VmxStructsAlloc(PVMCC pVM)
1928	{
1929	/*
1930	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1931	* The VMCS size cannot be more than 4096 bytes.
1932	*
1933	* See Intel spec. Appendix A.1 "Basic VMX Information".
1934	*/
1935	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1936	if (cbVmcs <= X86_PAGE_4K_SIZE)
1937	{ /* likely */ }
1938	else
1939	{
1940	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1941	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1942	}
1943
1944	/*
1945	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1946	*/
1947	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1948	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1949	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1950	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1951	#endif
1952
1953	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1954	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1955	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1956
1957	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1958	{
1959	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1960	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1961	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1962	}
1963
1964	/*
1965	* Allocate per-VM VT-x structures.
1966	*/
1967	int rc = VINF_SUCCESS;
1968	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1969	/* Allocate crash-dump magic scratch page. */
1970	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1971	if (RT_FAILURE(rc))
1972	{
1973	hmR0VmxStructsFree(pVM);
1974	return rc;
1975	}
1976	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1977	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1978	#endif
1979
1980	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1981	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1982	{
1983	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1984	&pVM->hm.s.vmx.HCPhysApicAccess);
1985	if (RT_FAILURE(rc))
1986	{
1987	hmR0VmxStructsFree(pVM);
1988	return rc;
1989	}
1990	}
1991
1992	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1993	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1994	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1995	{
1996	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1997	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1998	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1999	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
2000	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
2001	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
2002	{
2003	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
2004	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
2005	if (RT_SUCCESS(rc))
2006	{
2007	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
2008	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
2009	}
2010	}
2011	else
2012	rc = VERR_NO_MEMORY;
2013
2014	if (RT_FAILURE(rc))
2015	{
2016	hmR0VmxStructsFree(pVM);
2017	return rc;
2018	}
2019	}
2020	#endif
2021
2022	/*
2023	* Initialize per-VCPU VT-x structures.
2024	*/
2025	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2026	{
2027	/* Allocate the guest VMCS structures. */
2028	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
2029	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2030	if (RT_SUCCESS(rc))
2031	{
2032	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2033	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2034	if (pVM->cpum.ro.GuestFeatures.fVmx)
2035	{
2036	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2037	if (RT_SUCCESS(rc))
2038	{ /* likely */ }
2039	else
2040	break;
2041	}
2042	#endif
2043	}
2044	else
2045	break;
2046	}
2047
2048	if (RT_FAILURE(rc))
2049	{
2050	hmR0VmxStructsFree(pVM);
2051	return rc;
2052	}
2053
2054	return VINF_SUCCESS;
2055	}
2056
2057	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2058	/**
2059	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2060	*
2061	* @returns @c true if the MSR is intercepted, @c false otherwise.
2062	* @param pvMsrBitmap The MSR bitmap.
2063	* @param offMsr The MSR byte offset.
2064	* @param iBit The bit offset from the byte offset.
2065	*/
2066	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2067	{
2068	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2069	Assert(pbMsrBitmap);
2070	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2071	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2072	}
2073	#endif
2074
2075	/**
2076	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2077	*
2078	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2079	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2080	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2081	* the read/write access of this MSR.
2082	*
2083	* @param pVCpu The cross context virtual CPU structure.
2084	* @param pVmcsInfo The VMCS info. object.
2085	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2086	* @param idMsr The MSR value.
2087	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2088	* include both a read -and- a write permission!
2089	*
2090	* @sa CPUMGetVmxMsrPermission.
2091	* @remarks Can be called with interrupts disabled.
2092	*/
2093	static void hmR0VmxSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2094	{
2095	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2096	Assert(pbMsrBitmap);
2097	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2098
2099	/*
2100	* MSR-bitmap Layout:
2101	* Byte index MSR range Interpreted as
2102	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2103	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2104	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2105	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2106	*
2107	* A bit corresponding to an MSR within the above range causes a VM-exit
2108	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2109	* the MSR range, it always cause a VM-exit.
2110	*
2111	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2112	*/
2113	uint16_t const offBitmapRead = 0;
2114	uint16_t const offBitmapWrite = 0x800;
2115	uint16_t offMsr;
2116	int32_t iBit;
2117	if (idMsr <= UINT32_C(0x00001fff))
2118	{
2119	offMsr = 0;
2120	iBit = idMsr;
2121	}
2122	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2123	{
2124	offMsr = 0x400;
2125	iBit = idMsr - UINT32_C(0xc0000000);
2126	}
2127	else
2128	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2129
2130	/*
2131	* Set the MSR read permission.
2132	*/
2133	uint16_t const offMsrRead = offBitmapRead + offMsr;
2134	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2135	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2136	{
2137	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2138	bool const fClear = !fIsNstGstVmcs ? true
2139	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2140	#else
2141	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2142	bool const fClear = true;
2143	#endif
2144	if (fClear)
2145	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2146	}
2147	else
2148	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2149
2150	/*
2151	* Set the MSR write permission.
2152	*/
2153	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2154	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2155	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2156	{
2157	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2158	bool const fClear = !fIsNstGstVmcs ? true
2159	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2160	#else
2161	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2162	bool const fClear = true;
2163	#endif
2164	if (fClear)
2165	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2166	}
2167	else
2168	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2169	}
2170
2171
2172	/**
2173	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2174	* area.
2175	*
2176	* @returns VBox status code.
2177	* @param pVCpu The cross context virtual CPU structure.
2178	* @param pVmcsInfo The VMCS info. object.
2179	* @param cMsrs The number of MSRs.
2180	*/
2181	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2182	{
2183	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2184	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2185	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2186	{
2187	/* Commit the MSR counts to the VMCS and update the cache. */
2188	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2189	{
2190	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2191	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
2192	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2193	pVmcsInfo->cEntryMsrLoad = cMsrs;
2194	pVmcsInfo->cExitMsrStore = cMsrs;
2195	pVmcsInfo->cExitMsrLoad = cMsrs;
2196	}
2197	return VINF_SUCCESS;
2198	}
2199
2200	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2201	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2202	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2203	}
2204
2205
2206	/**
2207	* Adds a new (or updates the value of an existing) guest/host MSR
2208	* pair to be swapped during the world-switch as part of the
2209	* auto-load/store MSR area in the VMCS.
2210	*
2211	* @returns VBox status code.
2212	* @param pVCpu The cross context virtual CPU structure.
2213	* @param pVmxTransient The VMX-transient structure.
2214	* @param idMsr The MSR.
2215	* @param uGuestMsrValue Value of the guest MSR.
2216	* @param fSetReadWrite Whether to set the guest read/write access of this
2217	* MSR (thus not causing a VM-exit).
2218	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2219	* necessary.
2220	*/
2221	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2222	bool fSetReadWrite, bool fUpdateHostMsr)
2223	{
2224	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2225	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2226	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2227	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2228	uint32_t i;
2229
2230	/* Paranoia. */
2231	Assert(pGuestMsrLoad);
2232
2233	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2234
2235	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2236	for (i = 0; i < cMsrs; i++)
2237	{
2238	if (pGuestMsrLoad[i].u32Msr == idMsr)
2239	break;
2240	}
2241
2242	bool fAdded = false;
2243	if (i == cMsrs)
2244	{
2245	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2246	++cMsrs;
2247	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2248	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2249
2250	/* Set the guest to read/write this MSR without causing VM-exits. */
2251	if ( fSetReadWrite
2252	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2253	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2254
2255	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2256	fAdded = true;
2257	}
2258
2259	/* Update the MSR value for the newly added or already existing MSR. */
2260	pGuestMsrLoad[i].u32Msr = idMsr;
2261	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2262
2263	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2264	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2265	{
2266	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2267	pGuestMsrStore[i].u32Msr = idMsr;
2268	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2269	}
2270
2271	/* Update the corresponding slot in the host MSR area. */
2272	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2273	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2274	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2275	pHostMsr[i].u32Msr = idMsr;
2276
2277	/*
2278	* Only if the caller requests to update the host MSR value AND we've newly added the
2279	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2280	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2281	*
2282	* We do this for performance reasons since reading MSRs may be quite expensive.
2283	*/
2284	if (fAdded)
2285	{
2286	if (fUpdateHostMsr)
2287	{
2288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2290	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2291	}
2292	else
2293	{
2294	/* Someone else can do the work. */
2295	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2296	}
2297	}
2298	return VINF_SUCCESS;
2299	}
2300
2301
2302	/**
2303	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2304	* auto-load/store MSR area in the VMCS.
2305	*
2306	* @returns VBox status code.
2307	* @param pVCpu The cross context virtual CPU structure.
2308	* @param pVmxTransient The VMX-transient structure.
2309	* @param idMsr The MSR.
2310	*/
2311	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2312	{
2313	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2314	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2315	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2316	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2317
2318	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2319
2320	for (uint32_t i = 0; i < cMsrs; i++)
2321	{
2322	/* Find the MSR. */
2323	if (pGuestMsrLoad[i].u32Msr == idMsr)
2324	{
2325	/*
2326	* If it's the last MSR, we only need to reduce the MSR count.
2327	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2328	*/
2329	if (i < cMsrs - 1)
2330	{
2331	/* Remove it from the VM-entry MSR-load area. */
2332	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2333	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2334
2335	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2336	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2337	{
2338	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2339	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2340	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2341	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2342	}
2343
2344	/* Remove it from the VM-exit MSR-load area. */
2345	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2346	Assert(pHostMsr[i].u32Msr == idMsr);
2347	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2348	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2349	}
2350
2351	/* Reduce the count to reflect the removed MSR and bail. */
2352	--cMsrs;
2353	break;
2354	}
2355	}
2356
2357	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2358	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2359	{
2360	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2361	AssertRCReturn(rc, rc);
2362
2363	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2364	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2365	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2366
2367	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2368	return VINF_SUCCESS;
2369	}
2370
2371	return VERR_NOT_FOUND;
2372	}
2373
2374
2375	/**
2376	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2377	*
2378	* @returns @c true if found, @c false otherwise.
2379	* @param pVmcsInfo The VMCS info. object.
2380	* @param idMsr The MSR to find.
2381	*/
2382	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2383	{
2384	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2385	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2386	Assert(pMsrs);
2387	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2388	for (uint32_t i = 0; i < cMsrs; i++)
2389	{
2390	if (pMsrs[i].u32Msr == idMsr)
2391	return true;
2392	}
2393	return false;
2394	}
2395
2396
2397	/**
2398	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2399	*
2400	* @param pVCpu The cross context virtual CPU structure.
2401	* @param pVmcsInfo The VMCS info. object.
2402	*
2403	* @remarks No-long-jump zone!!!
2404	*/
2405	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2406	{
2407	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2408
2409	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2410	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2411	Assert(pHostMsrLoad);
2412	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2413	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2414	for (uint32_t i = 0; i < cMsrs; i++)
2415	{
2416	/*
2417	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2418	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2419	*/
2420	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2421	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2422	else
2423	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2424	}
2425	}
2426
2427
2428	/**
2429	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2430	* perform lazy restoration of the host MSRs while leaving VT-x.
2431	*
2432	* @param pVCpu The cross context virtual CPU structure.
2433	*
2434	* @remarks No-long-jump zone!!!
2435	*/
2436	static void hmR0VmxLazySaveHostMsrs(PVMCPUCC pVCpu)
2437	{
2438	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2439
2440	/*
2441	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2442	*/
2443	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2444	{
2445	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2446	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2447	{
2448	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2449	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2450	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2451	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2452	}
2453	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2454	}
2455	}
2456
2457
2458	/**
2459	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2460	* lazily while leaving VT-x.
2461	*
2462	* @returns true if it does, false otherwise.
2463	* @param pVCpu The cross context virtual CPU structure.
2464	* @param idMsr The MSR to check.
2465	*/
2466	static bool hmR0VmxIsLazyGuestMsr(PCVMCPUCC pVCpu, uint32_t idMsr)
2467	{
2468	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2469	{
2470	switch (idMsr)
2471	{
2472	case MSR_K8_LSTAR:
2473	case MSR_K6_STAR:
2474	case MSR_K8_SF_MASK:
2475	case MSR_K8_KERNEL_GS_BASE:
2476	return true;
2477	}
2478	}
2479	return false;
2480	}
2481
2482
2483	/**
2484	* Loads a set of guests MSRs to allow read/passthru to the guest.
2485	*
2486	* The name of this function is slightly confusing. This function does NOT
2487	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2488	* common prefix for functions dealing with "lazy restoration" of the shared
2489	* MSRs.
2490	*
2491	* @param pVCpu The cross context virtual CPU structure.
2492	*
2493	* @remarks No-long-jump zone!!!
2494	*/
2495	static void hmR0VmxLazyLoadGuestMsrs(PVMCPUCC pVCpu)
2496	{
2497	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2498	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2499
2500	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2501	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2502	{
2503	/*
2504	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2505	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2506	* we can skip a few MSR writes.
2507	*
2508	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2509	* guest MSR values in the guest-CPU context might be different to what's currently
2510	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2511	* CPU, see @bugref{8728}.
2512	*/
2513	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2514	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2515	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2516	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2517	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2518	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2519	{
2520	#ifdef VBOX_STRICT
2521	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2522	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2523	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2524	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2525	#endif
2526	}
2527	else
2528	{
2529	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2530	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2531	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2532	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2533	}
2534	}
2535	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2536	}
2537
2538
2539	/**
2540	* Performs lazy restoration of the set of host MSRs if they were previously
2541	* loaded with guest MSR values.
2542	*
2543	* @param pVCpu The cross context virtual CPU structure.
2544	*
2545	* @remarks No-long-jump zone!!!
2546	* @remarks The guest MSRs should have been saved back into the guest-CPU
2547	* context by hmR0VmxImportGuestState()!!!
2548	*/
2549	static void hmR0VmxLazyRestoreHostMsrs(PVMCPUCC pVCpu)
2550	{
2551	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2552	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2553
2554	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2555	{
2556	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2557	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2558	{
2559	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2560	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2561	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2562	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2563	}
2564	}
2565	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2566	}
2567
2568
2569	/**
2570	* Verifies that our cached values of the VMCS fields are all consistent with
2571	* what's actually present in the VMCS.
2572	*
2573	* @returns VBox status code.
2574	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2575	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2576	* VMCS content. HMCPU error-field is
2577	* updated, see VMX_VCI_XXX.
2578	* @param pVCpu The cross context virtual CPU structure.
2579	* @param pVmcsInfo The VMCS info. object.
2580	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2581	*/
2582	static int hmR0VmxCheckVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2583	{
2584	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2585
2586	uint32_t u32Val;
2587	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2588	AssertRC(rc);
2589	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2590	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2591	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2592	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2593
2594	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2595	AssertRC(rc);
2596	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2597	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2598	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2599	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2600
2601	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2602	AssertRC(rc);
2603	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2604	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2605	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2606	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2607
2608	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2609	AssertRC(rc);
2610	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2611	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2612	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2613	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2614
2615	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2616	{
2617	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2618	AssertRC(rc);
2619	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2620	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2621	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2622	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2623	}
2624
2625	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2626	AssertRC(rc);
2627	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2628	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2629	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2630	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2631
2632	uint64_t u64Val;
2633	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2634	AssertRC(rc);
2635	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2636	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2637	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2638	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2639
2640	NOREF(pcszVmcs);
2641	return VINF_SUCCESS;
2642	}
2643
2644
2645	#ifdef VBOX_STRICT
2646	/**
2647	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2648	*
2649	* @param pVCpu The cross context virtual CPU structure.
2650	* @param pVmcsInfo The VMCS info. object.
2651	*/
2652	static void hmR0VmxCheckHostEferMsr(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2653	{
2654	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2655
2656	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2657	{
2658	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2659	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2660	uint64_t uVmcsEferMsrVmcs;
2661	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2662	AssertRC(rc);
2663
2664	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2665	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2666	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2667	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2668	}
2669	}
2670
2671
2672	/**
2673	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2674	* VMCS are correct.
2675	*
2676	* @param pVCpu The cross context virtual CPU structure.
2677	* @param pVmcsInfo The VMCS info. object.
2678	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2679	*/
2680	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2681	{
2682	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2683
2684	/* Read the various MSR-area counts from the VMCS. */
2685	uint32_t cEntryLoadMsrs;
2686	uint32_t cExitStoreMsrs;
2687	uint32_t cExitLoadMsrs;
2688	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2689	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2690	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2691
2692	/* Verify all the MSR counts are the same. */
2693	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2694	Assert(cExitStoreMsrs == cExitLoadMsrs);
2695	uint32_t const cMsrs = cExitLoadMsrs;
2696
2697	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2698	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2699
2700	/* Verify the MSR counts are within the allocated page size. */
2701	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2702
2703	/* Verify the relevant contents of the MSR areas match. */
2704	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2705	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2706	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2707	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2708	for (uint32_t i = 0; i < cMsrs; i++)
2709	{
2710	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2711	if (fSeparateExitMsrStorePage)
2712	{
2713	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2714	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2715	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2716	}
2717
2718	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2719	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2720	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2721
2722	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2723	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2724	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2725	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2726
2727	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2728	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2729	if (fIsEferMsr)
2730	{
2731	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2732	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2733	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2734	}
2735
2736	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2737	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2738	{
2739	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2740	if (fIsEferMsr)
2741	{
2742	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2743	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2744	}
2745	else
2746	{
2747	if (!fIsNstGstVmcs)
2748	{
2749	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2750	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2751	}
2752	else
2753	{
2754	/*
2755	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2756	* execute a nested-guest with MSR passthrough.
2757	*
2758	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2759	* allow passthrough too.
2760	*/
2761	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2762	Assert(pvMsrBitmapNstGst);
2763	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2764	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2765	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2766	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2767	}
2768	}
2769	}
2770
2771	/* Move to the next MSR. */
2772	pHostMsrLoad++;
2773	pGuestMsrLoad++;
2774	pGuestMsrStore++;
2775	}
2776	}
2777	#endif /* VBOX_STRICT */
2778
2779
2780	/**
2781	* Flushes the TLB using EPT.
2782	*
2783	* @returns VBox status code.
2784	* @param pVCpu The cross context virtual CPU structure of the calling
2785	* EMT. Can be NULL depending on @a enmTlbFlush.
2786	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2787	* enmTlbFlush.
2788	* @param enmTlbFlush Type of flush.
2789	*
2790	* @remarks Caller is responsible for making sure this function is called only
2791	* when NestedPaging is supported and providing @a enmTlbFlush that is
2792	* supported by the CPU.
2793	* @remarks Can be called with interrupts disabled.
2794	*/
2795	static void hmR0VmxFlushEpt(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2796	{
2797	uint64_t au64Descriptor[2];
2798	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2799	au64Descriptor[0] = 0;
2800	else
2801	{
2802	Assert(pVCpu);
2803	Assert(pVmcsInfo);
2804	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2805	}
2806	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2807
2808	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2809	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2810
2811	if ( RT_SUCCESS(rc)
2812	&& pVCpu)
2813	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2814	}
2815
2816
2817	/**
2818	* Flushes the TLB using VPID.
2819	*
2820	* @returns VBox status code.
2821	* @param pVCpu The cross context virtual CPU structure of the calling
2822	* EMT. Can be NULL depending on @a enmTlbFlush.
2823	* @param enmTlbFlush Type of flush.
2824	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2825	* on @a enmTlbFlush).
2826	*
2827	* @remarks Can be called with interrupts disabled.
2828	*/
2829	static void hmR0VmxFlushVpid(PVMCPUCC pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2830	{
2831	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2832
2833	uint64_t au64Descriptor[2];
2834	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2835	{
2836	au64Descriptor[0] = 0;
2837	au64Descriptor[1] = 0;
2838	}
2839	else
2840	{
2841	AssertPtr(pVCpu);
2842	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2843	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2844	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2845	au64Descriptor[1] = GCPtr;
2846	}
2847
2848	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2849	AssertMsg(rc == VINF_SUCCESS,
2850	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2851
2852	if ( RT_SUCCESS(rc)
2853	&& pVCpu)
2854	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2855	NOREF(rc);
2856	}
2857
2858
2859	/**
2860	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2861	* otherwise there is nothing really to invalidate.
2862	*
2863	* @returns VBox status code.
2864	* @param pVCpu The cross context virtual CPU structure.
2865	* @param GCVirt Guest virtual address of the page to invalidate.
2866	*/
2867	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
2868	{
2869	AssertPtr(pVCpu);
2870	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2871
2872	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2873	{
2874	/*
2875	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2876	* the EPT case. See @bugref{6043} and @bugref{6177}.
2877	*
2878	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2879	* as this function maybe called in a loop with individual addresses.
2880	*/
2881	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2882	if (pVM->hm.s.vmx.fVpid)
2883	{
2884	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2885	if (fVpidFlush)
2886	{
2887	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2888	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2889	}
2890	else
2891	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2892	}
2893	else if (pVM->hm.s.fNestedPaging)
2894	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2895	}
2896
2897	return VINF_SUCCESS;
2898	}
2899
2900
2901	/**
2902	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2903	* case where neither EPT nor VPID is supported by the CPU.
2904	*
2905	* @param pHostCpu The HM physical-CPU structure.
2906	* @param pVCpu The cross context virtual CPU structure.
2907	*
2908	* @remarks Called with interrupts disabled.
2909	*/
2910	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2911	{
2912	AssertPtr(pVCpu);
2913	AssertPtr(pHostCpu);
2914
2915	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2916
2917	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2918	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2919	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2920	pVCpu->hm.s.fForceTLBFlush = false;
2921	return;
2922	}
2923
2924
2925	/**
2926	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2927	*
2928	* @param pHostCpu The HM physical-CPU structure.
2929	* @param pVCpu The cross context virtual CPU structure.
2930	* @param pVmcsInfo The VMCS info. object.
2931	*
2932	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2933	* nomenclature. The reason is, to avoid confusion in compare statements
2934	* since the host-CPU copies are named "ASID".
2935	*
2936	* @remarks Called with interrupts disabled.
2937	*/
2938	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2939	{
2940	#ifdef VBOX_WITH_STATISTICS
2941	bool fTlbFlushed = false;
2942	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2943	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2944	if (!fTlbFlushed) \
2945	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2946	} while (0)
2947	#else
2948	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2949	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2950	#endif
2951
2952	AssertPtr(pVCpu);
2953	AssertPtr(pHostCpu);
2954	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2955
2956	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2957	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2958	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2959	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2960
2961	/*
2962	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2963	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2964	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2965	* cannot reuse the current ASID anymore.
2966	*/
2967	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2968	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2969	{
2970	++pHostCpu->uCurrentAsid;
2971	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2972	{
2973	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2974	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2975	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2976	}
2977
2978	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2979	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2980	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2981
2982	/*
2983	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2984	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2985	*/
2986	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2987	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2988	HMVMX_SET_TAGGED_TLB_FLUSHED();
2989	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2990	}
2991	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2992	{
2993	/*
2994	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2995	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2996	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2997	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2998	* mappings, see @bugref{6568}.
2999	*
3000	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
3001	*/
3002	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3003	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3004	HMVMX_SET_TAGGED_TLB_FLUSHED();
3005	}
3006	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3007	{
3008	/*
3009	* The nested-guest specifies its own guest-physical address to use as the APIC-access
3010	* address which requires flushing the TLB of EPT cached structures.
3011	*
3012	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
3013	*/
3014	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3015	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3016	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3017	HMVMX_SET_TAGGED_TLB_FLUSHED();
3018	}
3019
3020
3021	pVCpu->hm.s.fForceTLBFlush = false;
3022	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3023
3024	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3025	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3026	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3027	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3028	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3029	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3030	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3031	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3032	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3033
3034	/* Update VMCS with the VPID. */
3035	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3036	AssertRC(rc);
3037
3038	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3039	}
3040
3041
3042	/**
3043	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3044	*
3045	* @param pHostCpu The HM physical-CPU structure.
3046	* @param pVCpu The cross context virtual CPU structure.
3047	* @param pVmcsInfo The VMCS info. object.
3048	*
3049	* @remarks Called with interrupts disabled.
3050	*/
3051	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3052	{
3053	AssertPtr(pVCpu);
3054	AssertPtr(pHostCpu);
3055	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3056	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3057	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3058
3059	/*
3060	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3061	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3062	*/
3063	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3064	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3065	{
3066	pVCpu->hm.s.fForceTLBFlush = true;
3067	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3068	}
3069
3070	/* Check for explicit TLB flushes. */
3071	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3072	{
3073	pVCpu->hm.s.fForceTLBFlush = true;
3074	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3075	}
3076
3077	/* Check for TLB flushes while switching to/from a nested-guest. */
3078	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3079	{
3080	pVCpu->hm.s.fForceTLBFlush = true;
3081	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3082	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3083	}
3084
3085	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3086	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3087
3088	if (pVCpu->hm.s.fForceTLBFlush)
3089	{
3090	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3091	pVCpu->hm.s.fForceTLBFlush = false;
3092	}
3093	}
3094
3095
3096	/**
3097	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3098	*
3099	* @param pHostCpu The HM physical-CPU structure.
3100	* @param pVCpu The cross context virtual CPU structure.
3101	*
3102	* @remarks Called with interrupts disabled.
3103	*/
3104	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
3105	{
3106	AssertPtr(pVCpu);
3107	AssertPtr(pHostCpu);
3108	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3109	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3110	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3111
3112	/*
3113	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3114	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3115	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3116	* cannot reuse the current ASID anymore.
3117	*/
3118	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3119	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3120	{
3121	pVCpu->hm.s.fForceTLBFlush = true;
3122	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3123	}
3124
3125	/* Check for explicit TLB flushes. */
3126	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3127	{
3128	/*
3129	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3130	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3131	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3132	* include fExplicitFlush's too) - an obscure corner case.
3133	*/
3134	pVCpu->hm.s.fForceTLBFlush = true;
3135	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3136	}
3137
3138	/* Check for TLB flushes while switching to/from a nested-guest. */
3139	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3140	{
3141	pVCpu->hm.s.fForceTLBFlush = true;
3142	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3143	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3144	}
3145
3146	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3147	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3148	if (pVCpu->hm.s.fForceTLBFlush)
3149	{
3150	++pHostCpu->uCurrentAsid;
3151	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3152	{
3153	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3154	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3155	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3156	}
3157
3158	pVCpu->hm.s.fForceTLBFlush = false;
3159	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3160	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3161	if (pHostCpu->fFlushAsidBeforeUse)
3162	{
3163	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3164	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3165	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3166	{
3167	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3168	pHostCpu->fFlushAsidBeforeUse = false;
3169	}
3170	else
3171	{
3172	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3173	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3174	}
3175	}
3176	}
3177
3178	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3179	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3180	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3181	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3182	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3183	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3184	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3185
3186	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3187	AssertRC(rc);
3188	}
3189
3190
3191	/**
3192	* Flushes the guest TLB entry based on CPU capabilities.
3193	*
3194	* @param pHostCpu The HM physical-CPU structure.
3195	* @param pVCpu The cross context virtual CPU structure.
3196	* @param pVmcsInfo The VMCS info. object.
3197	*
3198	* @remarks Called with interrupts disabled.
3199	*/
3200	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3201	{
3202	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3203	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3204	#endif
3205	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3206	switch (pVM->hm.s.vmx.enmTlbFlushType)
3207	{
3208	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3209	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3210	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3211	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3212	default:
3213	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3214	break;
3215	}
3216	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3217	}
3218
3219
3220	/**
3221	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3222	* TLB entries from the host TLB before VM-entry.
3223	*
3224	* @returns VBox status code.
3225	* @param pVM The cross context VM structure.
3226	*/
3227	static int hmR0VmxSetupTaggedTlb(PVMCC pVM)
3228	{
3229	/*
3230	* Determine optimal flush type for nested paging.
3231	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3232	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3233	*/
3234	if (pVM->hm.s.fNestedPaging)
3235	{
3236	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3237	{
3238	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3239	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3240	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3241	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3242	else
3243	{
3244	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3245	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3246	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3247	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3248	}
3249
3250	/* Make sure the write-back cacheable memory type for EPT is supported. */
3251	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3252	{
3253	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3254	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3255	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3256	}
3257
3258	/* EPT requires a page-walk length of 4. */
3259	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3260	{
3261	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3262	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3263	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3264	}
3265	}
3266	else
3267	{
3268	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3269	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3270	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3271	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3272	}
3273	}
3274
3275	/*
3276	* Determine optimal flush type for VPID.
3277	*/
3278	if (pVM->hm.s.vmx.fVpid)
3279	{
3280	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3281	{
3282	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3283	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3284	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3285	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3286	else
3287	{
3288	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3289	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3290	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3291	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3292	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3293	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3294	pVM->hm.s.vmx.fVpid = false;
3295	}
3296	}
3297	else
3298	{
3299	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3300	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3301	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3302	pVM->hm.s.vmx.fVpid = false;
3303	}
3304	}
3305
3306	/*
3307	* Setup the handler for flushing tagged-TLBs.
3308	*/
3309	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3310	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3311	else if (pVM->hm.s.fNestedPaging)
3312	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3313	else if (pVM->hm.s.vmx.fVpid)
3314	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3315	else
3316	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3317	return VINF_SUCCESS;
3318	}
3319
3320
3321	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3322	/**
3323	* Sets up the shadow VMCS fields arrays.
3324	*
3325	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3326	* executing the guest.
3327	*
3328	* @returns VBox status code.
3329	* @param pVM The cross context VM structure.
3330	*/
3331	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVMCC pVM)
3332	{
3333	/*
3334	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3335	* when the host does not support it.
3336	*/
3337	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3338	if ( !fGstVmwriteAll
3339	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3340	{ /* likely. */ }
3341	else
3342	{
3343	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3344	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3345	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3346	}
3347
3348	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3349	uint32_t cRwFields = 0;
3350	uint32_t cRoFields = 0;
3351	for (uint32_t i = 0; i < cVmcsFields; i++)
3352	{
3353	VMXVMCSFIELD VmcsField;
3354	VmcsField.u = g_aVmcsFields[i];
3355
3356	/*
3357	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3358	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3359	* in the shadow VMCS fields array as they would be redundant.
3360	*
3361	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3362	* we must not include it in the shadow VMCS fields array. Guests attempting to
3363	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3364	* the required behavior.
3365	*/
3366	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3367	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3368	{
3369	/*
3370	* Read-only fields are placed in a separate array so that while syncing shadow
3371	* VMCS fields later (which is more performance critical) we can avoid branches.
3372	*
3373	* However, if the guest can write to all fields (including read-only fields),
3374	* we treat it a as read/write field. Otherwise, writing to these fields would
3375	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3376	*/
3377	if ( fGstVmwriteAll
3378	\|\| !VMXIsVmcsFieldReadOnly(VmcsField.u))
3379	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3380	else
3381	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3382	}
3383	}
3384
3385	/* Update the counts. */
3386	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3387	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3388	return VINF_SUCCESS;
3389	}
3390
3391
3392	/**
3393	* Sets up the VMREAD and VMWRITE bitmaps.
3394	*
3395	* @param pVM The cross context VM structure.
3396	*/
3397	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVMCC pVM)
3398	{
3399	/*
3400	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3401	*/
3402	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3403	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3404	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3405	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3406	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3407
3408	/*
3409	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3410	* VMREAD and VMWRITE bitmaps.
3411	*/
3412	{
3413	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3414	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3415	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3416	{
3417	uint32_t const uVmcsField = paShadowVmcsFields[i];
3418	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3419	Assert(uVmcsField >> 3 < cbBitmap);
3420	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3421	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3422	}
3423	}
3424
3425	/*
3426	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3427	* if the host supports VMWRITE to all supported VMCS fields.
3428	*/
3429	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3430	{
3431	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3432	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3433	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3434	{
3435	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3436	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3437	Assert(uVmcsField >> 3 < cbBitmap);
3438	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3439	}
3440	}
3441	}
3442	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3443
3444
3445	/**
3446	* Sets up the virtual-APIC page address for the VMCS.
3447	*
3448	* @param pVmcsInfo The VMCS info. object.
3449	*/
3450	DECLINLINE(void) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
3451	{
3452	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3453	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3454	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3455	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3456	AssertRC(rc);
3457	}
3458
3459
3460	/**
3461	* Sets up the MSR-bitmap address for the VMCS.
3462	*
3463	* @param pVmcsInfo The VMCS info. object.
3464	*/
3465	DECLINLINE(void) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
3466	{
3467	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3468	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3469	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3470	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3471	AssertRC(rc);
3472	}
3473
3474
3475	/**
3476	* Sets up the APIC-access page address for the VMCS.
3477	*
3478	* @param pVCpu The cross context virtual CPU structure.
3479	*/
3480	DECLINLINE(void) hmR0VmxSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
3481	{
3482	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3483	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3484	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3485	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3486	AssertRC(rc);
3487	}
3488
3489
3490	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3491	/**
3492	* Sets up the VMREAD bitmap address for the VMCS.
3493	*
3494	* @param pVCpu The cross context virtual CPU structure.
3495	*/
3496	DECLINLINE(void) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
3497	{
3498	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3499	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3500	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3501	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3502	AssertRC(rc);
3503	}
3504
3505
3506	/**
3507	* Sets up the VMWRITE bitmap address for the VMCS.
3508	*
3509	* @param pVCpu The cross context virtual CPU structure.
3510	*/
3511	DECLINLINE(void) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
3512	{
3513	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3514	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3515	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3516	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3517	AssertRC(rc);
3518	}
3519	#endif
3520
3521
3522	/**
3523	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3524	* in the VMCS.
3525	*
3526	* @returns VBox status code.
3527	* @param pVmcsInfo The VMCS info. object.
3528	*/
3529	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
3530	{
3531	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3532	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3533	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3534
3535	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3536	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3537	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3538
3539	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3540	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3541	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3542
3543	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad); AssertRC(rc);
3544	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore); AssertRC(rc);
3545	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad); AssertRC(rc);
3546	return VINF_SUCCESS;
3547	}
3548
3549
3550	/**
3551	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3552	*
3553	* @param pVCpu The cross context virtual CPU structure.
3554	* @param pVmcsInfo The VMCS info. object.
3555	*/
3556	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3557	{
3558	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3559
3560	/*
3561	* The guest can access the following MSRs (read, write) without causing
3562	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3563	*/
3564	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3565	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3566	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3567	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3568	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3569	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3570
3571	/*
3572	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3573	* associated with then. We never need to intercept access (writes need to be
3574	* executed without causing a VM-exit, reads will #GP fault anyway).
3575	*
3576	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3577	* read/write them. We swap the the guest/host MSR value using the
3578	* auto-load/store MSR area.
3579	*/
3580	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3581	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3582	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3583	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3584	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3585	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3586
3587	/*
3588	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3589	* required for 64-bit guests.
3590	*/
3591	if (pVM->hm.s.fAllow64BitGuests)
3592	{
3593	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3594	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3595	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3596	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3597	}
3598
3599	/*
3600	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3601	*/
3602	#ifdef VBOX_STRICT
3603	Assert(pVmcsInfo->pvMsrBitmap);
3604	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3605	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3606	#endif
3607	}
3608
3609
3610	/**
3611	* Sets up pin-based VM-execution controls in the VMCS.
3612	*
3613	* @returns VBox status code.
3614	* @param pVCpu The cross context virtual CPU structure.
3615	* @param pVmcsInfo The VMCS info. object.
3616	*/
3617	static int hmR0VmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3618	{
3619	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3620	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3621	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3622
3623	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3624	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3625
3626	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3627	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3628
3629	/* Enable the VMX-preemption timer. */
3630	if (pVM->hm.s.vmx.fUsePreemptTimer)
3631	{
3632	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3633	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3634	}
3635
3636	#if 0
3637	/* Enable posted-interrupt processing. */
3638	if (pVM->hm.s.fPostedIntrs)
3639	{
3640	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3641	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3642	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3643	}
3644	#endif
3645
3646	if ((fVal & fZap) != fVal)
3647	{
3648	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3649	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3650	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3651	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3652	}
3653
3654	/* Commit it to the VMCS and update our cache. */
3655	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3656	AssertRC(rc);
3657	pVmcsInfo->u32PinCtls = fVal;
3658
3659	return VINF_SUCCESS;
3660	}
3661
3662
3663	/**
3664	* Sets up secondary processor-based VM-execution controls in the VMCS.
3665	*
3666	* @returns VBox status code.
3667	* @param pVCpu The cross context virtual CPU structure.
3668	* @param pVmcsInfo The VMCS info. object.
3669	*/
3670	static int hmR0VmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3671	{
3672	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3673	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3674	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3675
3676	/* WBINVD causes a VM-exit. */
3677	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3678	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3679
3680	/* Enable EPT (aka nested-paging). */
3681	if (pVM->hm.s.fNestedPaging)
3682	fVal \|= VMX_PROC_CTLS2_EPT;
3683
3684	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3685	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3686	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3687	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3688	fVal \|= VMX_PROC_CTLS2_INVPCID;
3689
3690	/* Enable VPID. */
3691	if (pVM->hm.s.vmx.fVpid)
3692	fVal \|= VMX_PROC_CTLS2_VPID;
3693
3694	/* Enable unrestricted guest execution. */
3695	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3696	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3697
3698	#if 0
3699	if (pVM->hm.s.fVirtApicRegs)
3700	{
3701	/* Enable APIC-register virtualization. */
3702	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3703	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3704
3705	/* Enable virtual-interrupt delivery. */
3706	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3707	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3708	}
3709	#endif
3710
3711	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3712	where the TPR shadow resides. */
3713	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3714	* done dynamically. */
3715	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3716	{
3717	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3718	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3719	}
3720
3721	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3722	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3723	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3724	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3725	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3726
3727	/* Enable Pause-Loop exiting. */
3728	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3729	&& pVM->hm.s.vmx.cPleGapTicks
3730	&& pVM->hm.s.vmx.cPleWindowTicks)
3731	{
3732	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3733
3734	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
3735	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
3736	}
3737
3738	if ((fVal & fZap) != fVal)
3739	{
3740	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3741	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3742	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3743	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3744	}
3745
3746	/* Commit it to the VMCS and update our cache. */
3747	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3748	AssertRC(rc);
3749	pVmcsInfo->u32ProcCtls2 = fVal;
3750
3751	return VINF_SUCCESS;
3752	}
3753
3754
3755	/**
3756	* Sets up processor-based VM-execution controls in the VMCS.
3757	*
3758	* @returns VBox status code.
3759	* @param pVCpu The cross context virtual CPU structure.
3760	* @param pVmcsInfo The VMCS info. object.
3761	*/
3762	static int hmR0VmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3763	{
3764	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3765
3766	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3767	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3768
3769	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3770	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3771	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3772	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3773	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3774	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3775	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3776
3777	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3778	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3779	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3780	{
3781	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3782	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3783	}
3784
3785	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3786	if (!pVM->hm.s.fNestedPaging)
3787	{
3788	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3789	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3790	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3791	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3792	}
3793
3794	/* Use TPR shadowing if supported by the CPU. */
3795	if ( PDMHasApic(pVM)
3796	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3797	{
3798	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3799	/* CR8 writes cause a VM-exit based on TPR threshold. */
3800	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3801	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3802	hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3803	}
3804	else
3805	{
3806	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3807	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3808	if (pVM->hm.s.fAllow64BitGuests)
3809	{
3810	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3811	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3812	}
3813	}
3814
3815	/* Use MSR-bitmaps if supported by the CPU. */
3816	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3817	{
3818	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3819	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3820	}
3821
3822	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3823	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3824	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3825
3826	if ((fVal & fZap) != fVal)
3827	{
3828	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3829	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3830	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3831	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3832	}
3833
3834	/* Commit it to the VMCS and update our cache. */
3835	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3836	AssertRC(rc);
3837	pVmcsInfo->u32ProcCtls = fVal;
3838
3839	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3840	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3841	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3842
3843	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3844	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3845	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3846
3847	/* Sanity check, should not really happen. */
3848	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3849	{ /* likely */ }
3850	else
3851	{
3852	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3853	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3854	}
3855
3856	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3857	return VINF_SUCCESS;
3858	}
3859
3860
3861	/**
3862	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3863	* Processor-based VM-execution) control fields in the VMCS.
3864	*
3865	* @returns VBox status code.
3866	* @param pVCpu The cross context virtual CPU structure.
3867	* @param pVmcsInfo The VMCS info. object.
3868	*/
3869	static int hmR0VmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3870	{
3871	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3872	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3873	{
3874	hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3875	hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3876	}
3877	#endif
3878
3879	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3880	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3881	AssertRC(rc);
3882
3883	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3884	if (RT_SUCCESS(rc))
3885	{
3886	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3887	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3888
3889	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
3890	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
3891
3892	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3893	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3894	return VINF_SUCCESS;
3895	}
3896	else
3897	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3898	return rc;
3899	}
3900
3901
3902	/**
3903	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3904	*
3905	* We shall setup those exception intercepts that don't change during the
3906	* lifetime of the VM here. The rest are done dynamically while loading the
3907	* guest state.
3908	*
3909	* @param pVCpu The cross context virtual CPU structure.
3910	* @param pVmcsInfo The VMCS info. object.
3911	*/
3912	static void hmR0VmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3913	{
3914	/*
3915	* The following exceptions are always intercepted:
3916	*
3917	* #AC - To prevent the guest from hanging the CPU.
3918	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3919	* recursive #DBs can cause a CPU hang.
3920	* #PF - To sync our shadow page tables when nested-paging is not used.
3921	*/
3922	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3923	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3924	\| RT_BIT(X86_XCPT_DB)
3925	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3926
3927	/* Commit it to the VMCS. */
3928	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3929	AssertRC(rc);
3930
3931	/* Update our cache of the exception bitmap. */
3932	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3933	}
3934
3935
3936	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3937	/**
3938	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3939	*
3940	* @returns VBox status code.
3941	* @param pVCpu The cross context virtual CPU structure.
3942	* @param pVmcsInfo The VMCS info. object.
3943	*/
3944	static int hmR0VmxSetupVmcsCtlsNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3945	{
3946	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3947	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3948	AssertRC(rc);
3949
3950	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3951	if (RT_SUCCESS(rc))
3952	{
3953	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3954	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3955
3956	/* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
3957	Assert(!pVmcsInfo->u64Cr0Mask);
3958	Assert(!pVmcsInfo->u64Cr4Mask);
3959	return VINF_SUCCESS;
3960	}
3961	else
3962	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3963	return rc;
3964	}
3965	#endif
3966
3967
3968	/**
3969	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3970	* VMX.
3971	*
3972	* @returns VBox status code.
3973	* @param pVCpu The cross context virtual CPU structure.
3974	* @param pVmcsInfo The VMCS info. object.
3975	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3976	*/
3977	static int hmR0VmxSetupVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3978	{
3979	Assert(pVmcsInfo->pvVmcs);
3980	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3981
3982	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3983	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3984	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3985	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3986
3987	LogFlowFunc(("\n"));
3988
3989	/*
3990	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3991	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3992	*/
3993	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3994	if (RT_SUCCESS(rc))
3995	{
3996	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3997	if (RT_SUCCESS(rc))
3998	{
3999	if (!fIsNstGstVmcs)
4000	{
4001	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
4002	if (RT_SUCCESS(rc))
4003	{
4004	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4005	if (RT_SUCCESS(rc))
4006	{
4007	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4008	if (RT_SUCCESS(rc))
4009	{
4010	hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4011	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4012	/*
4013	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4014	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4015	* making it fit for use when VMCS shadowing is later enabled.
4016	*/
4017	if (pVmcsInfo->pvShadowVmcs)
4018	{
4019	VMXVMCSREVID VmcsRevId;
4020	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4021	VmcsRevId.n.fIsShadowVmcs = 1;
4022	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4023	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4024	if (RT_SUCCESS(rc))
4025	{ /* likely */ }
4026	else
4027	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4028	}
4029	#endif
4030	}
4031	else
4032	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4033	}
4034	else
4035	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4036	}
4037	else
4038	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4039	}
4040	else
4041	{
4042	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4043	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4044	if (RT_SUCCESS(rc))
4045	{ /* likely */ }
4046	else
4047	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4048	#else
4049	AssertFailed();
4050	#endif
4051	}
4052	}
4053	else
4054	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4055	}
4056	else
4057	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4058
4059	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4060	if (RT_SUCCESS(rc))
4061	{
4062	rc = hmR0VmxClearVmcs(pVmcsInfo);
4063	if (RT_SUCCESS(rc))
4064	{ /* likely */ }
4065	else
4066	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4067	}
4068
4069	/*
4070	* Update the last-error record both for failures and success, so we
4071	* can propagate the status code back to ring-3 for diagnostics.
4072	*/
4073	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4074	NOREF(pszVmcs);
4075	return rc;
4076	}
4077
4078
4079	/**
4080	* Does global VT-x initialization (called during module initialization).
4081	*
4082	* @returns VBox status code.
4083	*/
4084	VMMR0DECL(int) VMXR0GlobalInit(void)
4085	{
4086	#ifdef HMVMX_USE_FUNCTION_TABLE
4087	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4088	# ifdef VBOX_STRICT
4089	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4090	Assert(g_apfnVMExitHandlers[i]);
4091	# endif
4092	#endif
4093	return VINF_SUCCESS;
4094	}
4095
4096
4097	/**
4098	* Does global VT-x termination (called during module termination).
4099	*/
4100	VMMR0DECL(void) VMXR0GlobalTerm()
4101	{
4102	/* Nothing to do currently. */
4103	}
4104
4105
4106	/**
4107	* Sets up and activates VT-x on the current CPU.
4108	*
4109	* @returns VBox status code.
4110	* @param pHostCpu The HM physical-CPU structure.
4111	* @param pVM The cross context VM structure. Can be
4112	* NULL after a host resume operation.
4113	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4114	* fEnabledByHost is @c true).
4115	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4116	* @a fEnabledByHost is @c true).
4117	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4118	* enable VT-x on the host.
4119	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4120	*/
4121	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4122	PCSUPHWVIRTMSRS pHwvirtMsrs)
4123	{
4124	AssertPtr(pHostCpu);
4125	AssertPtr(pHwvirtMsrs);
4126	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4127
4128	/* Enable VT-x if it's not already enabled by the host. */
4129	if (!fEnabledByHost)
4130	{
4131	int rc = hmR0VmxEnterRootMode(pHostCpu, pVM, HCPhysCpuPage, pvCpuPage);
4132	if (RT_FAILURE(rc))
4133	return rc;
4134	}
4135
4136	/*
4137	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4138	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4139	* invalidated when flushing by VPID.
4140	*/
4141	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4142	{
4143	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4144	pHostCpu->fFlushAsidBeforeUse = false;
4145	}
4146	else
4147	pHostCpu->fFlushAsidBeforeUse = true;
4148
4149	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4150	++pHostCpu->cTlbFlushes;
4151
4152	return VINF_SUCCESS;
4153	}
4154
4155
4156	/**
4157	* Deactivates VT-x on the current CPU.
4158	*
4159	* @returns VBox status code.
4160	* @param pHostCpu The HM physical-CPU structure.
4161	* @param pvCpuPage Pointer to the VMXON region.
4162	* @param HCPhysCpuPage Physical address of the VMXON region.
4163	*
4164	* @remarks This function should never be called when SUPR0EnableVTx() or
4165	* similar was used to enable VT-x on the host.
4166	*/
4167	VMMR0DECL(int) VMXR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4168	{
4169	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4170
4171	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4172	return hmR0VmxLeaveRootMode(pHostCpu);
4173	}
4174
4175
4176	/**
4177	* Does per-VM VT-x initialization.
4178	*
4179	* @returns VBox status code.
4180	* @param pVM The cross context VM structure.
4181	*/
4182	VMMR0DECL(int) VMXR0InitVM(PVMCC pVM)
4183	{
4184	AssertPtr(pVM);
4185	LogFlowFunc(("pVM=%p\n", pVM));
4186
4187	int rc = hmR0VmxStructsAlloc(pVM);
4188	if (RT_FAILURE(rc))
4189	{
4190	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4191	return rc;
4192	}
4193
4194	return VINF_SUCCESS;
4195	}
4196
4197
4198	/**
4199	* Does per-VM VT-x termination.
4200	*
4201	* @returns VBox status code.
4202	* @param pVM The cross context VM structure.
4203	*/
4204	VMMR0DECL(int) VMXR0TermVM(PVMCC pVM)
4205	{
4206	AssertPtr(pVM);
4207	LogFlowFunc(("pVM=%p\n", pVM));
4208
4209	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4210	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4211	{
4212	Assert(pVM->hm.s.vmx.pvScratch);
4213	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4214	}
4215	#endif
4216	hmR0VmxStructsFree(pVM);
4217	return VINF_SUCCESS;
4218	}
4219
4220
4221	/**
4222	* Sets up the VM for execution using hardware-assisted VMX.
4223	* This function is only called once per-VM during initialization.
4224	*
4225	* @returns VBox status code.
4226	* @param pVM The cross context VM structure.
4227	*/
4228	VMMR0DECL(int) VMXR0SetupVM(PVMCC pVM)
4229	{
4230	AssertPtr(pVM);
4231	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4232
4233	LogFlowFunc(("pVM=%p\n", pVM));
4234
4235	/*
4236	* At least verify if VMX is enabled, since we can't check if we're in VMX root mode or not
4237	* without causing a #GP.
4238	*/
4239	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4240	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4241	{ /* likely */ }
4242	else
4243	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4244
4245	/*
4246	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4247	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4248	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4249	*/
4250	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4251	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4252	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4253	{
4254	LogRelFunc(("Invalid real-on-v86 state.\n"));
4255	return VERR_INTERNAL_ERROR;
4256	}
4257
4258	/* Initialize these always, see hmR3InitFinalizeR0().*/
4259	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4260	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4261
4262	/* Setup the tagged-TLB flush handlers. */
4263	int rc = hmR0VmxSetupTaggedTlb(pVM);
4264	if (RT_FAILURE(rc))
4265	{
4266	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4267	return rc;
4268	}
4269
4270	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4271	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4272	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4273	{
4274	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4275	if (RT_SUCCESS(rc))
4276	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4277	else
4278	{
4279	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4280	return rc;
4281	}
4282	}
4283	#endif
4284
4285	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4286	{
4287	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
4288	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4289
4290	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4291	if (RT_SUCCESS(rc))
4292	{
4293	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4294	if (pVM->cpum.ro.GuestFeatures.fVmx)
4295	{
4296	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4297	if (RT_SUCCESS(rc))
4298	{ /* likely */ }
4299	else
4300	{
4301	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4302	return rc;
4303	}
4304	}
4305	#endif
4306	}
4307	else
4308	{
4309	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4310	return rc;
4311	}
4312	}
4313
4314	return VINF_SUCCESS;
4315	}
4316
4317
4318	/**
4319	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4320	* the VMCS.
4321	*/
4322	static void hmR0VmxExportHostControlRegs(void)
4323	{
4324	int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, ASMGetCR0()); AssertRC(rc);
4325	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, ASMGetCR3()); AssertRC(rc);
4326	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, ASMGetCR4()); AssertRC(rc);
4327	}
4328
4329
4330	/**
4331	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4332	* the host-state area in the VMCS.
4333	*
4334	* @returns VBox status code.
4335	* @param pVCpu The cross context virtual CPU structure.
4336	*/
4337	static int hmR0VmxExportHostSegmentRegs(PVMCPUCC pVCpu)
4338	{
4339	/**
4340	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4341	* requirements. See hmR0VmxExportHostSegmentRegs().
4342	*/
4343	#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_selValue) \
4344	if ((a_selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4345	{ \
4346	bool fValidSelector = true; \
4347	if ((a_selValue) & X86_SEL_LDT) \
4348	{ \
4349	uint32_t const uAttr = ASMGetSegAttr(a_selValue); \
4350	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4351	} \
4352	if (fValidSelector) \
4353	{ \
4354	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##a_Seg; \
4355	pVCpu->hm.s.vmx.RestoreHost.uHostSel##a_Seg = (a_selValue); \
4356	} \
4357	(a_selValue) = 0; \
4358	}
4359
4360	/*
4361	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4362	* will be messed up. We should -not- save the messed up state without restoring
4363	* the original host-state, see @bugref{7240}.
4364	*
4365	* This apparently can happen (most likely the FPU changes), deal with it rather than
4366	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4367	*/
4368	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4369	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4370	{
4371	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4372	pVCpu->idCpu));
4373	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4374	}
4375	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4376
4377	/*
4378	* Host segment registers.
4379	*/
4380	RTSEL uSelES = ASMGetES();
4381	RTSEL uSelCS = ASMGetCS();
4382	RTSEL uSelSS = ASMGetSS();
4383	RTSEL uSelDS = ASMGetDS();
4384	RTSEL uSelFS = ASMGetFS();
4385	RTSEL uSelGS = ASMGetGS();
4386	RTSEL uSelTR = ASMGetTR();
4387
4388	/*
4389	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4390	* gain VM-entry and restore them before we get preempted.
4391	*
4392	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4393	*/
4394	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4395	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4396	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4397	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4398
4399	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4400	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4401	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4402	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4403	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4404	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4405	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4406	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4407	Assert(uSelCS);
4408	Assert(uSelTR);
4409
4410	/* Write these host selector fields into the host-state area in the VMCS. */
4411	int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS); AssertRC(rc);
4412	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS); AssertRC(rc);
4413	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS); AssertRC(rc);
4414	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES); AssertRC(rc);
4415	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS); AssertRC(rc);
4416	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS); AssertRC(rc);
4417	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR); AssertRC(rc);
4418
4419	/*
4420	* Host GDTR and IDTR.
4421	*/
4422	RTGDTR Gdtr;
4423	RTIDTR Idtr;
4424	RT_ZERO(Gdtr);
4425	RT_ZERO(Idtr);
4426	ASMGetGDTR(&Gdtr);
4427	ASMGetIDTR(&Idtr);
4428	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRC(rc);
4429	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRC(rc);
4430
4431	/*
4432	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4433	* them to the maximum limit (0xffff) on every VM-exit.
4434	*/
4435	if (Gdtr.cbGdt != 0xffff)
4436	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4437
4438	/*
4439	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4440	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4441	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4442	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4443	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4444	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4445	* at 0xffff on hosts where we are sure it won't cause trouble.
4446	*/
4447	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4448	if (Idtr.cbIdt < 0x0fff)
4449	#else
4450	if (Idtr.cbIdt != 0xffff)
4451	#endif
4452	{
4453	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4454	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4455	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4456	}
4457
4458	/*
4459	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4460	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4461	* RPL should be too in most cases.
4462	*/
4463	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4464	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4465
4466	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4467	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4468
4469	/*
4470	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4471	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4472	* restoration if the host has something else. Task switching is not supported in 64-bit
4473	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4474	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4475	*
4476	* [1] See Intel spec. 3.5 "System Descriptor Types".
4477	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4478	*/
4479	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4480	Assert(pDesc->System.u4Type == 11);
4481	if ( pDesc->System.u16LimitLow != 0x67
4482	\|\| pDesc->System.u4LimitHigh)
4483	{
4484	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4485	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4486	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4487	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4488	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4489	}
4490
4491	/*
4492	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4493	*/
4494	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4495	{
4496	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4497	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4498	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4499	{
4500	/* The GDT is read-only but the writable GDT is available. */
4501	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4502	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4503	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4504	AssertRCReturn(rc, rc);
4505	}
4506	}
4507
4508	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase);
4509	AssertRC(rc);
4510
4511	/*
4512	* Host FS base and GS base.
4513	*/
4514	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4515	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4516	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRC(rc);
4517	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRC(rc);
4518
4519	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4520	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4521	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4522	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4523	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4524
4525	return VINF_SUCCESS;
4526	#undef VMXLOCAL_ADJUST_HOST_SEG
4527	}
4528
4529
4530	/**
4531	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4532	* host-state area of the VMCS.
4533	*
4534	* These MSRs will be automatically restored on the host after every successful
4535	* VM-exit.
4536	*
4537	* @param pVCpu The cross context virtual CPU structure.
4538	*
4539	* @remarks No-long-jump zone!!!
4540	*/
4541	static void hmR0VmxExportHostMsrs(PVMCPUCC pVCpu)
4542	{
4543	AssertPtr(pVCpu);
4544
4545	/*
4546	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4547	* rather than swapping them on every VM-entry.
4548	*/
4549	hmR0VmxLazySaveHostMsrs(pVCpu);
4550
4551	/*
4552	* Host Sysenter MSRs.
4553	*/
4554	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)); AssertRC(rc);
4555	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP)); AssertRC(rc);
4556	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP)); AssertRC(rc);
4557
4558	/*
4559	* Host EFER MSR.
4560	*
4561	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4562	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4563	*/
4564	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4565	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4566	{
4567	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4568	AssertRC(rc);
4569	}
4570
4571	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4572	* hmR0VmxExportGuestEntryExitCtls(). */
4573	}
4574
4575
4576	/**
4577	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4578	*
4579	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4580	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4581	*
4582	* @returns true if we need to load guest EFER, false otherwise.
4583	* @param pVCpu The cross context virtual CPU structure.
4584	* @param pVmxTransient The VMX-transient structure.
4585	*
4586	* @remarks Requires EFER, CR4.
4587	* @remarks No-long-jump zone!!!
4588	*/
4589	static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4590	{
4591	#ifdef HMVMX_ALWAYS_SWAP_EFER
4592	RT_NOREF2(pVCpu, pVmxTransient);
4593	return true;
4594	#else
4595	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4596	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4597	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4598	uint64_t const u64GuestEfer = pCtx->msrEFER;
4599
4600	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4601	/*
4602	* For nested-guests, we shall honor swapping the EFER MSR when requested by
4603	* the nested-guest.
4604	*/
4605	if ( pVmxTransient->fIsNestedGuest
4606	&& ( CPUMIsGuestVmxEntryCtlsSet(pVCpu, pCtx, VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4607	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_SAVE_EFER_MSR)
4608	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_LOAD_EFER_MSR)))
4609	return true;
4610	# else
4611	RT_NOREF(pVmxTransient);
4612	#endif
4613
4614	/*
4615	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4616	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4617	*/
4618	if ( CPUMIsGuestInLongModeEx(pCtx)
4619	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4620	return true;
4621
4622	/*
4623	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4624	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4625	*
4626	* See Intel spec. 4.5 "IA-32e Paging".
4627	* See Intel spec. 4.1.1 "Three Paging Modes".
4628	*
4629	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4630	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4631	*/
4632	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4633	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4634	if ( (pCtx->cr4 & X86_CR4_PAE)
4635	&& (pCtx->cr0 & X86_CR0_PG))
4636	{
4637	/*
4638	* If nested paging is not used, verify that the guest paging mode matches the
4639	* shadow paging mode which is/will be placed in the VMCS (which is what will
4640	* actually be used while executing the guest and not the CR4 shadow value).
4641	*/
4642	AssertMsg(pVM->hm.s.fNestedPaging \|\| ( pVCpu->hm.s.enmShadowMode == PGMMODE_PAE
4643	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_PAE_NX
4644	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64
4645	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64_NX),
4646	("enmShadowMode=%u\n", pVCpu->hm.s.enmShadowMode));
4647	if ((u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4648	{
4649	/* Verify that the host is NX capable. */
4650	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4651	return true;
4652	}
4653	}
4654
4655	return false;
4656	#endif
4657	}
4658
4659
4660	/**
4661	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4662	* VMCS.
4663	*
4664	* This is typically required when the guest changes paging mode.
4665	*
4666	* @returns VBox status code.
4667	* @param pVCpu The cross context virtual CPU structure.
4668	* @param pVmxTransient The VMX-transient structure.
4669	*
4670	* @remarks Requires EFER.
4671	* @remarks No-long-jump zone!!!
4672	*/
4673	static int hmR0VmxExportGuestEntryExitCtls(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4674	{
4675	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4676	{
4677	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4678	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4679
4680	/*
4681	* VM-entry controls.
4682	*/
4683	{
4684	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4685	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4686
4687	/*
4688	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4689	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4690	*
4691	* For nested-guests, this is a mandatory VM-entry control. It's also
4692	* required because we do not want to leak host bits to the nested-guest.
4693	*/
4694	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4695
4696	/*
4697	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4698	*
4699	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4700	* required to get the nested-guest working with hardware-assisted VMX execution.
4701	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
4702	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4703	* here rather than while merging the guest VMCS controls.
4704	*/
4705	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4706	{
4707	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
4708	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4709	}
4710	else
4711	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4712
4713	/*
4714	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4715	*
4716	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4717	* regardless of whether the nested-guest VMCS specifies it because we are free to
4718	* load whatever MSRs we require and we do not need to modify the guest visible copy
4719	* of the VM-entry MSR load area.
4720	*/
4721	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4722	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4723	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4724	else
4725	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4726
4727	/*
4728	* The following should -not- be set (since we're not in SMM mode):
4729	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4730	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4731	*/
4732
4733	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4734	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4735
4736	if ((fVal & fZap) == fVal)
4737	{ /* likely */ }
4738	else
4739	{
4740	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4741	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4742	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4743	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4744	}
4745
4746	/* Commit it to the VMCS. */
4747	if (pVmcsInfo->u32EntryCtls != fVal)
4748	{
4749	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4750	AssertRC(rc);
4751	pVmcsInfo->u32EntryCtls = fVal;
4752	}
4753	}
4754
4755	/*
4756	* VM-exit controls.
4757	*/
4758	{
4759	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4760	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4761
4762	/*
4763	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4764	* supported the 1-setting of this bit.
4765	*
4766	* For nested-guests, we set the "save debug controls" as the converse
4767	* "load debug controls" is mandatory for nested-guests anyway.
4768	*/
4769	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4770
4771	/*
4772	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4773	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4774	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4775	* hmR0VmxExportHostMsrs().
4776	*
4777	* For nested-guests, we always set this bit as we do not support 32-bit
4778	* hosts.
4779	*/
4780	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4781
4782	/*
4783	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4784	*
4785	* For nested-guests, we should use the "save IA32_EFER" control if we also
4786	* used the "load IA32_EFER" control while exporting VM-entry controls.
4787	*/
4788	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4789	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4790	{
4791	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4792	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4793	}
4794
4795	/*
4796	* Enable saving of the VMX-preemption timer value on VM-exit.
4797	* For nested-guests, currently not exposed/used.
4798	*/
4799	if ( pVM->hm.s.vmx.fUsePreemptTimer
4800	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4801	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4802
4803	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4804	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4805
4806	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4807	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4808	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4809
4810	if ((fVal & fZap) == fVal)
4811	{ /* likely */ }
4812	else
4813	{
4814	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4815	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4816	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4817	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4818	}
4819
4820	/* Commit it to the VMCS. */
4821	if (pVmcsInfo->u32ExitCtls != fVal)
4822	{
4823	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4824	AssertRC(rc);
4825	pVmcsInfo->u32ExitCtls = fVal;
4826	}
4827	}
4828
4829	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4830	}
4831	return VINF_SUCCESS;
4832	}
4833
4834
4835	/**
4836	* Sets the TPR threshold in the VMCS.
4837	*
4838	* @param pVmcsInfo The VMCS info. object.
4839	* @param u32TprThreshold The TPR threshold (task-priority class only).
4840	*/
4841	DECLINLINE(void) hmR0VmxApicSetTprThreshold(PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4842	{
4843	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4844	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4845	RT_NOREF(pVmcsInfo);
4846	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4847	AssertRC(rc);
4848	}
4849
4850
4851	/**
4852	* Exports the guest APIC TPR state into the VMCS.
4853	*
4854	* @returns VBox status code.
4855	* @param pVCpu The cross context virtual CPU structure.
4856	* @param pVmxTransient The VMX-transient structure.
4857	*
4858	* @remarks No-long-jump zone!!!
4859	*/
4860	static int hmR0VmxExportGuestApicTpr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4861	{
4862	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4863	{
4864	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4865
4866	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4867	if (!pVmxTransient->fIsNestedGuest)
4868	{
4869	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4870	&& APICIsEnabled(pVCpu))
4871	{
4872	/*
4873	* Setup TPR shadowing.
4874	*/
4875	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4876	{
4877	bool fPendingIntr = false;
4878	uint8_t u8Tpr = 0;
4879	uint8_t u8PendingIntr = 0;
4880	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4881	AssertRCReturn(rc, rc);
4882
4883	/*
4884	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4885	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4886	* priority of the pending interrupt so we can deliver the interrupt. If there
4887	* are no interrupts pending, set threshold to 0 to not cause any
4888	* TPR-below-threshold VM-exits.
4889	*/
4890	uint32_t u32TprThreshold = 0;
4891	if (fPendingIntr)
4892	{
4893	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4894	(which is the Task-Priority Class). */
4895	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4896	const uint8_t u8TprPriority = u8Tpr >> 4;
4897	if (u8PendingPriority <= u8TprPriority)
4898	u32TprThreshold = u8PendingPriority;
4899	}
4900
4901	hmR0VmxApicSetTprThreshold(pVmcsInfo, u32TprThreshold);
4902	}
4903	}
4904	}
4905	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4906	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4907	}
4908	return VINF_SUCCESS;
4909	}
4910
4911
4912	/**
4913	* Gets the guest interruptibility-state.
4914	*
4915	* @returns Guest's interruptibility-state.
4916	* @param pVCpu The cross context virtual CPU structure.
4917	* @param pVmxTransient The VMX-transient structure.
4918	*
4919	* @remarks No-long-jump zone!!!
4920	*/
4921	static uint32_t hmR0VmxGetGuestIntrState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4922	{
4923	/*
4924	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4925	*/
4926	uint32_t fIntrState = 0;
4927	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4928	{
4929	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
4930	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
4931
4932	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4933	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4934	{
4935	if (pCtx->eflags.Bits.u1IF)
4936	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4937	else
4938	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4939	}
4940	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4941	{
4942	/*
4943	* We can clear the inhibit force flag as even if we go back to the recompiler
4944	* without executing guest code in VT-x, the flag's condition to be cleared is
4945	* met and thus the cleared state is correct.
4946	*/
4947	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4948	}
4949	}
4950
4951	/*
4952	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4953	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4954	* setting this would block host-NMIs and IRET will not clear the blocking.
4955	*
4956	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4957	*
4958	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4959	*/
4960	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4961	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4962	&& CPUMIsGuestNmiBlocking(pVCpu))
4963	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4964
4965	return fIntrState;
4966	}
4967
4968
4969	/**
4970	* Exports the exception intercepts required for guest execution in the VMCS.
4971	*
4972	* @returns VBox status code.
4973	* @param pVCpu The cross context virtual CPU structure.
4974	* @param pVmxTransient The VMX-transient structure.
4975	*
4976	* @remarks No-long-jump zone!!!
4977	*/
4978	static int hmR0VmxExportGuestXcptIntercepts(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4979	{
4980	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4981	{
4982	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4983	if ( !pVmxTransient->fIsNestedGuest
4984	&& pVCpu->hm.s.fGIMTrapXcptUD)
4985	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4986	else
4987	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4988
4989	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4990	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4991	}
4992	return VINF_SUCCESS;
4993	}
4994
4995
4996	/**
4997	* Exports the guest's RIP into the guest-state area in the VMCS.
4998	*
4999	* @returns VBox status code.
5000	* @param pVCpu The cross context virtual CPU structure.
5001	*
5002	* @remarks No-long-jump zone!!!
5003	*/
5004	static int hmR0VmxExportGuestRip(PVMCPUCC pVCpu)
5005	{
5006	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5007	{
5008	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5009
5010	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5011	AssertRC(rc);
5012
5013	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5014	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5015	}
5016	return VINF_SUCCESS;
5017	}
5018
5019
5020	/**
5021	* Exports the guest's RSP into the guest-state area in the VMCS.
5022	*
5023	* @returns VBox status code.
5024	* @param pVCpu The cross context virtual CPU structure.
5025	*
5026	* @remarks No-long-jump zone!!!
5027	*/
5028	static int hmR0VmxExportGuestRsp(PVMCPUCC pVCpu)
5029	{
5030	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5031	{
5032	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5033
5034	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5035	AssertRC(rc);
5036
5037	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5038	Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
5039	}
5040	return VINF_SUCCESS;
5041	}
5042
5043
5044	/**
5045	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5046	*
5047	* @returns VBox status code.
5048	* @param pVCpu The cross context virtual CPU structure.
5049	* @param pVmxTransient The VMX-transient structure.
5050	*
5051	* @remarks No-long-jump zone!!!
5052	*/
5053	static int hmR0VmxExportGuestRflags(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5054	{
5055	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5056	{
5057	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5058
5059	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5060	Let us assert it as such and use 32-bit VMWRITE. */
5061	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5062	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5063	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5064	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5065
5066	/*
5067	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5068	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5069	* can run the real-mode guest code under Virtual 8086 mode.
5070	*/
5071	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5072	if (pVmcsInfo->RealMode.fRealOnV86Active)
5073	{
5074	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5075	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5076	Assert(!pVmxTransient->fIsNestedGuest);
5077	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5078	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5079	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5080	}
5081
5082	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5083	AssertRC(rc);
5084
5085	/*
5086	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5087	*
5088	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5089	* through the hypervisor debugger using EFLAGS.TF.
5090	*/
5091	if ( !pVmxTransient->fIsNestedGuest
5092	&& !pVCpu->hm.s.fSingleInstruction
5093	&& fEFlags.Bits.u1TF)
5094	{
5095	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5096	* premature trips to ring-3 esp since IEM does not yet handle it. */
5097	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5098	AssertRC(rc);
5099	}
5100	/* else: for nested-guest currently handling while merging controls. */
5101
5102	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5103	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5104	}
5105	return VINF_SUCCESS;
5106	}
5107
5108
5109	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5110	/**
5111	* Copies the nested-guest VMCS to the shadow VMCS.
5112	*
5113	* @returns VBox status code.
5114	* @param pVCpu The cross context virtual CPU structure.
5115	* @param pVmcsInfo The VMCS info. object.
5116	*
5117	* @remarks No-long-jump zone!!!
5118	*/
5119	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5120	{
5121	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5122	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5123
5124	/*
5125	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5126	* current VMCS, as we may try saving guest lazy MSRs.
5127	*
5128	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5129	* calling the import VMCS code which is currently performing the guest MSR reads
5130	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5131	* and the rest of the VMX leave session machinery.
5132	*/
5133	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5134
5135	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5136	if (RT_SUCCESS(rc))
5137	{
5138	/*
5139	* Copy all guest read/write VMCS fields.
5140	*
5141	* We don't check for VMWRITE failures here for performance reasons and
5142	* because they are not expected to fail, barring irrecoverable conditions
5143	* like hardware errors.
5144	*/
5145	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5146	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5147	{
5148	uint64_t u64Val;
5149	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5150	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5151	VMXWriteVmcs64(uVmcsField, u64Val);
5152	}
5153
5154	/*
5155	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5156	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5157	*/
5158	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5159	{
5160	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5161	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5162	{
5163	uint64_t u64Val;
5164	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5165	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5166	VMXWriteVmcs64(uVmcsField, u64Val);
5167	}
5168	}
5169
5170	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5171	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5172	}
5173
5174	ASMSetFlags(fEFlags);
5175	return rc;
5176	}
5177
5178
5179	/**
5180	* Copies the shadow VMCS to the nested-guest VMCS.
5181	*
5182	* @returns VBox status code.
5183	* @param pVCpu The cross context virtual CPU structure.
5184	* @param pVmcsInfo The VMCS info. object.
5185	*
5186	* @remarks Called with interrupts disabled.
5187	*/
5188	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5189	{
5190	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5191	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5192	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5193
5194	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5195	if (RT_SUCCESS(rc))
5196	{
5197	/*
5198	* Copy guest read/write fields from the shadow VMCS.
5199	* Guest read-only fields cannot be modified, so no need to copy them.
5200	*
5201	* We don't check for VMREAD failures here for performance reasons and
5202	* because they are not expected to fail, barring irrecoverable conditions
5203	* like hardware errors.
5204	*/
5205	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5206	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5207	{
5208	uint64_t u64Val;
5209	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5210	VMXReadVmcs64(uVmcsField, &u64Val);
5211	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5212	}
5213
5214	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5215	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5216	}
5217	return rc;
5218	}
5219
5220
5221	/**
5222	* Enables VMCS shadowing for the given VMCS info. object.
5223	*
5224	* @param pVmcsInfo The VMCS info. object.
5225	*
5226	* @remarks No-long-jump zone!!!
5227	*/
5228	static void hmR0VmxEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5229	{
5230	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5231	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5232	{
5233	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5234	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5235	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5236	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
5237	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5238	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5239	Log4Func(("Enabled\n"));
5240	}
5241	}
5242
5243
5244	/**
5245	* Disables VMCS shadowing for the given VMCS info. object.
5246	*
5247	* @param pVmcsInfo The VMCS info. object.
5248	*
5249	* @remarks No-long-jump zone!!!
5250	*/
5251	static void hmR0VmxDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5252	{
5253	/*
5254	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5255	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5256	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5257	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5258	*
5259	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5260	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5261	*/
5262	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5263	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5264	{
5265	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5266	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5267	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
5268	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5269	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5270	Log4Func(("Disabled\n"));
5271	}
5272	}
5273	#endif
5274
5275
5276	/**
5277	* Exports the guest hardware-virtualization state.
5278	*
5279	* @returns VBox status code.
5280	* @param pVCpu The cross context virtual CPU structure.
5281	* @param pVmxTransient The VMX-transient structure.
5282	*
5283	* @remarks No-long-jump zone!!!
5284	*/
5285	static int hmR0VmxExportGuestHwvirtState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5286	{
5287	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5288	{
5289	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5290	/*
5291	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5292	* VMCS shadowing.
5293	*/
5294	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5295	{
5296	/*
5297	* If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
5298	* copy the nested hypervisor's current VMCS into the shadow VMCS and enable
5299	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5300	*
5301	* We check for VMX root mode here in case the guest executes VMXOFF without
5302	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5303	* not clear the current VMCS pointer.
5304	*/
5305	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5306	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5307	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5308	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5309	{
5310	/* Paranoia. */
5311	Assert(!pVmxTransient->fIsNestedGuest);
5312
5313	/*
5314	* For performance reasons, also check if the nested hypervisor's current VMCS
5315	* was newly loaded or modified before copying it to the shadow VMCS.
5316	*/
5317	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5318	{
5319	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5320	AssertRCReturn(rc, rc);
5321	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5322	}
5323	hmR0VmxEnableVmcsShadowing(pVmcsInfo);
5324	}
5325	else
5326	hmR0VmxDisableVmcsShadowing(pVmcsInfo);
5327	}
5328	#else
5329	NOREF(pVmxTransient);
5330	#endif
5331	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5332	}
5333	return VINF_SUCCESS;
5334	}
5335
5336
5337	/**
5338	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5339	*
5340	* The guest FPU state is always pre-loaded hence we don't need to bother about
5341	* sharing FPU related CR0 bits between the guest and host.
5342	*
5343	* @returns VBox status code.
5344	* @param pVCpu The cross context virtual CPU structure.
5345	* @param pVmxTransient The VMX-transient structure.
5346	*
5347	* @remarks No-long-jump zone!!!
5348	*/
5349	static int hmR0VmxExportGuestCR0(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5350	{
5351	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5352	{
5353	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5354	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5355
5356	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0;
5357	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5358	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5359	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5360	else
5361	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5362
5363	if (!pVmxTransient->fIsNestedGuest)
5364	{
5365	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5366	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5367	uint64_t const u64ShadowCr0 = u64GuestCr0;
5368	Assert(!RT_HI_U32(u64GuestCr0));
5369
5370	/*
5371	* Setup VT-x's view of the guest CR0.
5372	*/
5373	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5374	if (pVM->hm.s.fNestedPaging)
5375	{
5376	if (CPUMIsGuestPagingEnabled(pVCpu))
5377	{
5378	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5379	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5380	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5381	}
5382	else
5383	{
5384	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5385	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5386	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5387	}
5388
5389	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5390	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5391	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5392	}
5393	else
5394	{
5395	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5396	u64GuestCr0 \|= X86_CR0_WP;
5397	}
5398
5399	/*
5400	* Guest FPU bits.
5401	*
5402	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5403	* using CR0.TS.
5404	*
5405	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5406	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5407	*/
5408	u64GuestCr0 \|= X86_CR0_NE;
5409
5410	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5411	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5412
5413	/*
5414	* Update exception intercepts.
5415	*/
5416	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5417	if (pVmcsInfo->RealMode.fRealOnV86Active)
5418	{
5419	Assert(PDMVmmDevHeapIsEnabled(pVM));
5420	Assert(pVM->hm.s.vmx.pRealModeTSS);
5421	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5422	}
5423	else
5424	{
5425	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5426	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5427	if (fInterceptMF)
5428	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5429	}
5430
5431	/* Additional intercepts for debugging, define these yourself explicitly. */
5432	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5433	uXcptBitmap \|= 0
5434	\| RT_BIT(X86_XCPT_BP)
5435	\| RT_BIT(X86_XCPT_DE)
5436	\| RT_BIT(X86_XCPT_NM)
5437	\| RT_BIT(X86_XCPT_TS)
5438	\| RT_BIT(X86_XCPT_UD)
5439	\| RT_BIT(X86_XCPT_NP)
5440	\| RT_BIT(X86_XCPT_SS)
5441	\| RT_BIT(X86_XCPT_GP)
5442	\| RT_BIT(X86_XCPT_PF)
5443	\| RT_BIT(X86_XCPT_MF)
5444	;
5445	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5446	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5447	#endif
5448	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5449	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5450	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5451
5452	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5453	u64GuestCr0 \|= fSetCr0;
5454	u64GuestCr0 &= fZapCr0;
5455	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5456
5457	/* Commit the CR0 and related fields to the guest VMCS. */
5458	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5459	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5460	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5461	{
5462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5463	AssertRC(rc);
5464	}
5465	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5466	{
5467	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5468	AssertRC(rc);
5469	}
5470
5471	/* Update our caches. */
5472	pVmcsInfo->u32ProcCtls = uProcCtls;
5473	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5474
5475	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5476	}
5477	else
5478	{
5479	/*
5480	* With nested-guests, we may have extended the guest/host mask here since we
5481	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5482	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
5483	* originally supplied. We must copy those bits from the nested-guest CR0 into
5484	* the nested-guest CR0 read-shadow.
5485	*/
5486	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5487	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5488	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5489	Assert(!RT_HI_U32(u64GuestCr0));
5490	Assert(u64GuestCr0 & X86_CR0_NE);
5491
5492	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5493	u64GuestCr0 \|= fSetCr0;
5494	u64GuestCr0 &= fZapCr0;
5495	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5496
5497	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5498	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5499	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5500
5501	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5502	}
5503
5504	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5505	}
5506
5507	return VINF_SUCCESS;
5508	}
5509
5510
5511	/**
5512	* Exports the guest control registers (CR3, CR4) into the guest-state area
5513	* in the VMCS.
5514	*
5515	* @returns VBox strict status code.
5516	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5517	* without unrestricted guest access and the VMMDev is not presently
5518	* mapped (e.g. EFI32).
5519	*
5520	* @param pVCpu The cross context virtual CPU structure.
5521	* @param pVmxTransient The VMX-transient structure.
5522	*
5523	* @remarks No-long-jump zone!!!
5524	*/
5525	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5526	{
5527	int rc = VINF_SUCCESS;
5528	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5529
5530	/*
5531	* Guest CR2.
5532	* It's always loaded in the assembler code. Nothing to do here.
5533	*/
5534
5535	/*
5536	* Guest CR3.
5537	*/
5538	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5539	{
5540	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5541
5542	if (pVM->hm.s.fNestedPaging)
5543	{
5544	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5545	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5546
5547	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5548	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5549	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5550	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5551
5552	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5553	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5554	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5555
5556	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5557	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5558	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5559	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5560	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5561	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5562	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5563
5564	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5565	AssertRC(rc);
5566
5567	uint64_t u64GuestCr3;
5568	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5569	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5570	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5571	{
5572	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5573	if (CPUMIsGuestInPAEModeEx(pCtx))
5574	{
5575	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5576	AssertRC(rc);
5577	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
5578	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
5579	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
5580	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
5581	}
5582
5583	/*
5584	* The guest's view of its CR3 is unblemished with nested paging when the
5585	* guest is using paging or we have unrestricted guest execution to handle
5586	* the guest when it's not using paging.
5587	*/
5588	u64GuestCr3 = pCtx->cr3;
5589	}
5590	else
5591	{
5592	/*
5593	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5594	* thinks it accesses physical memory directly, we use our identity-mapped
5595	* page table to map guest-linear to guest-physical addresses. EPT takes care
5596	* of translating it to host-physical addresses.
5597	*/
5598	RTGCPHYS GCPhys;
5599	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5600
5601	/* We obtain it here every time as the guest could have relocated this PCI region. */
5602	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5603	if (RT_SUCCESS(rc))
5604	{ /* likely */ }
5605	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5606	{
5607	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5608	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5609	}
5610	else
5611	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5612
5613	u64GuestCr3 = GCPhys;
5614	}
5615
5616	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
5617	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, u64GuestCr3);
5618	AssertRC(rc);
5619	}
5620	else
5621	{
5622	Assert(!pVmxTransient->fIsNestedGuest);
5623	/* Non-nested paging case, just use the hypervisor's CR3. */
5624	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5625
5626	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
5627	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5628	AssertRC(rc);
5629	}
5630
5631	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5632	}
5633
5634	/*
5635	* Guest CR4.
5636	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5637	*/
5638	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5639	{
5640	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5641	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5642
5643	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0;
5644	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5645
5646	/*
5647	* With nested-guests, we may have extended the guest/host mask here (since we
5648	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5649	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5650	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
5651	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5652	*/
5653	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5654	uint64_t u64GuestCr4 = pCtx->cr4;
5655	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5656	? pCtx->cr4
5657	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5658	Assert(!RT_HI_U32(u64GuestCr4));
5659
5660	/*
5661	* Setup VT-x's view of the guest CR4.
5662	*
5663	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5664	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5665	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5666	*
5667	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5668	*/
5669	if (pVmcsInfo->RealMode.fRealOnV86Active)
5670	{
5671	Assert(pVM->hm.s.vmx.pRealModeTSS);
5672	Assert(PDMVmmDevHeapIsEnabled(pVM));
5673	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5674	}
5675
5676	if (pVM->hm.s.fNestedPaging)
5677	{
5678	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5679	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5680	{
5681	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5682	u64GuestCr4 \|= X86_CR4_PSE;
5683	/* Our identity mapping is a 32-bit page directory. */
5684	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5685	}
5686	/* else use guest CR4.*/
5687	}
5688	else
5689	{
5690	Assert(!pVmxTransient->fIsNestedGuest);
5691
5692	/*
5693	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5694	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5695	*/
5696	switch (pVCpu->hm.s.enmShadowMode)
5697	{
5698	case PGMMODE_REAL: /* Real-mode. */
5699	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5700	case PGMMODE_32_BIT: /* 32-bit paging. */
5701	{
5702	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5703	break;
5704	}
5705
5706	case PGMMODE_PAE: /* PAE paging. */
5707	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5708	{
5709	u64GuestCr4 \|= X86_CR4_PAE;
5710	break;
5711	}
5712
5713	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5714	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5715	{
5716	#ifdef VBOX_WITH_64_BITS_GUESTS
5717	/* For our assumption in hmR0VmxShouldSwapEferMsr. */
5718	Assert(u64GuestCr4 & X86_CR4_PAE);
5719	break;
5720	#endif
5721	}
5722	default:
5723	AssertFailed();
5724	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5725	}
5726	}
5727
5728	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
5729	u64GuestCr4 \|= fSetCr4;
5730	u64GuestCr4 &= fZapCr4;
5731
5732	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5733	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
5734	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
5735
5736	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5737	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5738
5739	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5740
5741	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5742	}
5743	return rc;
5744	}
5745
5746
5747	/**
5748	* Exports the guest debug registers into the guest-state area in the VMCS.
5749	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5750	*
5751	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5752	*
5753	* @returns VBox status code.
5754	* @param pVCpu The cross context virtual CPU structure.
5755	* @param pVmxTransient The VMX-transient structure.
5756	*
5757	* @remarks No-long-jump zone!!!
5758	*/
5759	static int hmR0VmxExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5760	{
5761	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5762
5763	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5764	* stepping. */
5765	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5766	if (pVmxTransient->fIsNestedGuest)
5767	{
5768	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5769	AssertRC(rc);
5770
5771	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5772	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5773	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5774	AssertRC(rc);
5775	return VINF_SUCCESS;
5776	}
5777
5778	#ifdef VBOX_STRICT
5779	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5780	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5781	{
5782	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5783	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5784	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5785	}
5786	#endif
5787
5788	bool fSteppingDB = false;
5789	bool fInterceptMovDRx = false;
5790	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5791	if (pVCpu->hm.s.fSingleInstruction)
5792	{
5793	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5794	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5795	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5796	{
5797	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5798	Assert(fSteppingDB == false);
5799	}
5800	else
5801	{
5802	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5803	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5804	pVCpu->hm.s.fClearTrapFlag = true;
5805	fSteppingDB = true;
5806	}
5807	}
5808
5809	uint64_t u64GuestDr7;
5810	if ( fSteppingDB
5811	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5812	{
5813	/*
5814	* Use the combined guest and host DRx values found in the hypervisor register set
5815	* because the hypervisor debugger has breakpoints active or someone is single stepping
5816	* on the host side without a monitor trap flag.
5817	*
5818	* Note! DBGF expects a clean DR6 state before executing guest code.
5819	*/
5820	if (!CPUMIsHyperDebugStateActive(pVCpu))
5821	{
5822	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5823	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5824	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5825	}
5826
5827	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5828	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
5829	pVCpu->hm.s.fUsingHyperDR7 = true;
5830	fInterceptMovDRx = true;
5831	}
5832	else
5833	{
5834	/*
5835	* If the guest has enabled debug registers, we need to load them prior to
5836	* executing guest code so they'll trigger at the right time.
5837	*/
5838	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5839	{
5840	if (!CPUMIsGuestDebugStateActive(pVCpu))
5841	{
5842	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5843	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5844	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5845	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5846	}
5847	Assert(!fInterceptMovDRx);
5848	}
5849	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5850	{
5851	/*
5852	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5853	* must intercept #DB in order to maintain a correct DR6 guest value, and
5854	* because we need to intercept it to prevent nested #DBs from hanging the
5855	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5856	*/
5857	fInterceptMovDRx = true;
5858	}
5859
5860	/* Update DR7 with the actual guest value. */
5861	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5862	pVCpu->hm.s.fUsingHyperDR7 = false;
5863	}
5864
5865	if (fInterceptMovDRx)
5866	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5867	else
5868	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5869
5870	/*
5871	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5872	* monitor-trap flag and update our cache.
5873	*/
5874	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5875	{
5876	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5877	AssertRC(rc);
5878	pVmcsInfo->u32ProcCtls = uProcCtls;
5879	}
5880
5881	/*
5882	* Update guest DR7.
5883	*/
5884	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
5885	AssertRC(rc);
5886
5887	/*
5888	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5889	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5890	*
5891	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5892	*/
5893	if (fSteppingDB)
5894	{
5895	Assert(pVCpu->hm.s.fSingleInstruction);
5896	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5897
5898	uint32_t fIntrState = 0;
5899	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5900	AssertRC(rc);
5901
5902	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5903	{
5904	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5905	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5906	AssertRC(rc);
5907	}
5908	}
5909
5910	return VINF_SUCCESS;
5911	}
5912
5913
5914	#ifdef VBOX_STRICT
5915	/**
5916	* Strict function to validate segment registers.
5917	*
5918	* @param pVCpu The cross context virtual CPU structure.
5919	* @param pVmcsInfo The VMCS info. object.
5920	*
5921	* @remarks Will import guest CR0 on strict builds during validation of
5922	* segments.
5923	*/
5924	static void hmR0VmxValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5925	{
5926	/*
5927	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5928	*
5929	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5930	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5931	* unusable bit and doesn't change the guest-context value.
5932	*/
5933	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5934	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5935	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5936	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5937	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5938	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5939	{
5940	/* Protected mode checks */
5941	/* CS */
5942	Assert(pCtx->cs.Attr.n.u1Present);
5943	Assert(!(pCtx->cs.Attr.u & 0xf00));
5944	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5945	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5946	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5947	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5948	\|\| (pCtx->cs.Attr.n.u1Granularity));
5949	/* CS cannot be loaded with NULL in protected mode. */
5950	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5951	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5952	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5953	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5954	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5955	else
5956	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5957	/* SS */
5958	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5959	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5960	if ( !(pCtx->cr0 & X86_CR0_PE)
5961	\|\| pCtx->cs.Attr.n.u4Type == 3)
5962	{
5963	Assert(!pCtx->ss.Attr.n.u2Dpl);
5964	}
5965	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5966	{
5967	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5968	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5969	Assert(pCtx->ss.Attr.n.u1Present);
5970	Assert(!(pCtx->ss.Attr.u & 0xf00));
5971	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5972	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5973	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5974	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5975	\|\| (pCtx->ss.Attr.n.u1Granularity));
5976	}
5977	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5978	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5979	{
5980	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5981	Assert(pCtx->ds.Attr.n.u1Present);
5982	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5983	Assert(!(pCtx->ds.Attr.u & 0xf00));
5984	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5985	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5986	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5987	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5988	\|\| (pCtx->ds.Attr.n.u1Granularity));
5989	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5990	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5991	}
5992	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5993	{
5994	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5995	Assert(pCtx->es.Attr.n.u1Present);
5996	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5997	Assert(!(pCtx->es.Attr.u & 0xf00));
5998	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5999	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6000	\|\| !(pCtx->es.Attr.n.u1Granularity));
6001	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6002	\|\| (pCtx->es.Attr.n.u1Granularity));
6003	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6004	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6005	}
6006	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6007	{
6008	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6009	Assert(pCtx->fs.Attr.n.u1Present);
6010	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6011	Assert(!(pCtx->fs.Attr.u & 0xf00));
6012	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6013	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6014	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6015	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6016	\|\| (pCtx->fs.Attr.n.u1Granularity));
6017	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6018	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6019	}
6020	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6021	{
6022	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6023	Assert(pCtx->gs.Attr.n.u1Present);
6024	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6025	Assert(!(pCtx->gs.Attr.u & 0xf00));
6026	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6027	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6028	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6029	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6030	\|\| (pCtx->gs.Attr.n.u1Granularity));
6031	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6032	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6033	}
6034	/* 64-bit capable CPUs. */
6035	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6036	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6037	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6038	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6039	}
6040	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6041	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6042	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6043	{
6044	/* Real and v86 mode checks. */
6045	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6046	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6047	if (pVmcsInfo->RealMode.fRealOnV86Active)
6048	{
6049	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6050	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6051	}
6052	else
6053	{
6054	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6055	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6056	}
6057
6058	/* CS */
6059	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6060	Assert(pCtx->cs.u32Limit == 0xffff);
6061	Assert(u32CSAttr == 0xf3);
6062	/* SS */
6063	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6064	Assert(pCtx->ss.u32Limit == 0xffff);
6065	Assert(u32SSAttr == 0xf3);
6066	/* DS */
6067	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6068	Assert(pCtx->ds.u32Limit == 0xffff);
6069	Assert(u32DSAttr == 0xf3);
6070	/* ES */
6071	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6072	Assert(pCtx->es.u32Limit == 0xffff);
6073	Assert(u32ESAttr == 0xf3);
6074	/* FS */
6075	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6076	Assert(pCtx->fs.u32Limit == 0xffff);
6077	Assert(u32FSAttr == 0xf3);
6078	/* GS */
6079	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6080	Assert(pCtx->gs.u32Limit == 0xffff);
6081	Assert(u32GSAttr == 0xf3);
6082	/* 64-bit capable CPUs. */
6083	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6084	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6085	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6086	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6087	}
6088	}
6089	#endif /* VBOX_STRICT */
6090
6091
6092	/**
6093	* Exports a guest segment register into the guest-state area in the VMCS.
6094	*
6095	* @returns VBox status code.
6096	* @param pVCpu The cross context virtual CPU structure.
6097	* @param pVmcsInfo The VMCS info. object.
6098	* @param iSegReg The segment register number (X86_SREG_XXX).
6099	* @param pSelReg Pointer to the segment selector.
6100	*
6101	* @remarks No-long-jump zone!!!
6102	*/
6103	static int hmR0VmxExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6104	{
6105	Assert(iSegReg < X86_SREG_COUNT);
6106	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6107	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6108	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6109	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6110
6111	uint32_t u32Access = pSelReg->Attr.u;
6112	if (pVmcsInfo->RealMode.fRealOnV86Active)
6113	{
6114	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6115	u32Access = 0xf3;
6116	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6117	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6118	RT_NOREF_PV(pVCpu);
6119	}
6120	else
6121	{
6122	/*
6123	* The way to differentiate between whether this is really a null selector or was just
6124	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6125	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6126	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6127	* NULL selectors loaded in protected-mode have their attribute as 0.
6128	*/
6129	if (!u32Access)
6130	u32Access = X86DESCATTR_UNUSABLE;
6131	}
6132
6133	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6134	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6135	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6136
6137	/*
6138	* Commit it to the VMCS.
6139	*/
6140	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); AssertRC(rc);
6141	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); AssertRC(rc);
6142	rc = VMXWriteVmcsNw(idxBase, pSelReg->u64Base); AssertRC(rc);
6143	rc = VMXWriteVmcs32(idxAttr, u32Access); AssertRC(rc);
6144	return VINF_SUCCESS;
6145	}
6146
6147
6148	/**
6149	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6150	* area in the VMCS.
6151	*
6152	* @returns VBox status code.
6153	* @param pVCpu The cross context virtual CPU structure.
6154	* @param pVmxTransient The VMX-transient structure.
6155	*
6156	* @remarks Will import guest CR0 on strict builds during validation of
6157	* segments.
6158	* @remarks No-long-jump zone!!!
6159	*/
6160	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6161	{
6162	int rc = VERR_INTERNAL_ERROR_5;
6163	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6164	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6165	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6166
6167	/*
6168	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6169	*/
6170	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6171	{
6172	#ifdef VBOX_WITH_REM
6173	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6174	{
6175	Assert(!pVmxTransient->fIsNestedGuest);
6176	Assert(pVM->hm.s.vmx.pRealModeTSS);
6177	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6178	if ( pVmcsInfo->fWasInRealMode
6179	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6180	{
6181	/*
6182	* Notify the recompiler must flush its code-cache as the guest -may-
6183	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6184	*/
6185	REMFlushTBs(pVM);
6186	Log4Func(("Switch to protected mode detected!\n"));
6187	pVmcsInfo->fWasInRealMode = false;
6188	}
6189	}
6190	#endif
6191	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6192	{
6193	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6194	if (pVmcsInfo->RealMode.fRealOnV86Active)
6195	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6196	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6197	AssertRC(rc);
6198	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6199	}
6200
6201	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6202	{
6203	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6204	if (pVmcsInfo->RealMode.fRealOnV86Active)
6205	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6206	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6207	AssertRC(rc);
6208	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6209	}
6210
6211	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6212	{
6213	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6214	if (pVmcsInfo->RealMode.fRealOnV86Active)
6215	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6216	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6217	AssertRC(rc);
6218	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6219	}
6220
6221	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6222	{
6223	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6224	if (pVmcsInfo->RealMode.fRealOnV86Active)
6225	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6226	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6227	AssertRC(rc);
6228	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6229	}
6230
6231	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6232	{
6233	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6234	if (pVmcsInfo->RealMode.fRealOnV86Active)
6235	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6236	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6237	AssertRC(rc);
6238	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6239	}
6240
6241	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6242	{
6243	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6244	if (pVmcsInfo->RealMode.fRealOnV86Active)
6245	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6246	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6247	AssertRC(rc);
6248	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6249	}
6250
6251	#ifdef VBOX_STRICT
6252	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6253	#endif
6254	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6255	pCtx->cs.Attr.u));
6256	}
6257
6258	/*
6259	* Guest TR.
6260	*/
6261	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6262	{
6263	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6264
6265	/*
6266	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6267	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6268	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6269	*/
6270	uint16_t u16Sel;
6271	uint32_t u32Limit;
6272	uint64_t u64Base;
6273	uint32_t u32AccessRights;
6274	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6275	{
6276	u16Sel = pCtx->tr.Sel;
6277	u32Limit = pCtx->tr.u32Limit;
6278	u64Base = pCtx->tr.u64Base;
6279	u32AccessRights = pCtx->tr.Attr.u;
6280	}
6281	else
6282	{
6283	Assert(!pVmxTransient->fIsNestedGuest);
6284	Assert(pVM->hm.s.vmx.pRealModeTSS);
6285	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6286
6287	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6288	RTGCPHYS GCPhys;
6289	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6290	AssertRCReturn(rc, rc);
6291
6292	X86DESCATTR DescAttr;
6293	DescAttr.u = 0;
6294	DescAttr.n.u1Present = 1;
6295	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6296
6297	u16Sel = 0;
6298	u32Limit = HM_VTX_TSS_SIZE;
6299	u64Base = GCPhys;
6300	u32AccessRights = DescAttr.u;
6301	}
6302
6303	/* Validate. */
6304	Assert(!(u16Sel & RT_BIT(2)));
6305	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6306	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6307	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6308	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6309	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6310	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6311	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6312	Assert( (u32Limit & 0xfff) == 0xfff
6313	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6314	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6315	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6316
6317	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
6318	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
6319	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
6320	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
6321
6322	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6323	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6324	}
6325
6326	/*
6327	* Guest GDTR.
6328	*/
6329	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6330	{
6331	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6332
6333	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
6334	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
6335
6336	/* Validate. */
6337	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6338
6339	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6340	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6341	}
6342
6343	/*
6344	* Guest LDTR.
6345	*/
6346	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6347	{
6348	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6349
6350	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6351	uint32_t u32Access;
6352	if ( !pVmxTransient->fIsNestedGuest
6353	&& !pCtx->ldtr.Attr.u)
6354	u32Access = X86DESCATTR_UNUSABLE;
6355	else
6356	u32Access = pCtx->ldtr.Attr.u;
6357
6358	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
6359	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
6360	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
6361	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
6362
6363	/* Validate. */
6364	if (!(u32Access & X86DESCATTR_UNUSABLE))
6365	{
6366	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6367	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6368	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6369	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6370	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6371	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6372	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6373	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6374	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6375	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6376	}
6377
6378	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6379	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6380	}
6381
6382	/*
6383	* Guest IDTR.
6384	*/
6385	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6386	{
6387	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6388
6389	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
6390	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
6391
6392	/* Validate. */
6393	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6394
6395	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6396	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6397	}
6398
6399	return VINF_SUCCESS;
6400	}
6401
6402
6403	/**
6404	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6405	* areas.
6406	*
6407	* These MSRs will automatically be loaded to the host CPU on every successful
6408	* VM-entry and stored from the host CPU on every successful VM-exit.
6409	*
6410	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6411	* actual host MSR values are not- updated here for performance reasons. See
6412	* hmR0VmxExportHostMsrs().
6413	*
6414	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6415	*
6416	* @returns VBox status code.
6417	* @param pVCpu The cross context virtual CPU structure.
6418	* @param pVmxTransient The VMX-transient structure.
6419	*
6420	* @remarks No-long-jump zone!!!
6421	*/
6422	static int hmR0VmxExportGuestMsrs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6423	{
6424	AssertPtr(pVCpu);
6425	AssertPtr(pVmxTransient);
6426
6427	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6428	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6429
6430	/*
6431	* MSRs that we use the auto-load/store MSR area in the VMCS.
6432	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
6433	* nothing to do here. The host MSR values are updated when it's safe in
6434	* hmR0VmxLazySaveHostMsrs().
6435	*
6436	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6437	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6438	* emulation. The merged MSR permission bitmap will ensure that we get VM-exits
6439	* for any MSR that are not part of the lazy MSRs so we do not need to place
6440	* those MSRs into the auto-load/store MSR area. Nothing to do here.
6441	*/
6442	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6443	{
6444	/* No auto-load/store MSRs currently. */
6445	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6446	}
6447
6448	/*
6449	* Guest Sysenter MSRs.
6450	*/
6451	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6452	{
6453	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6454
6455	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6456	{
6457	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6458	AssertRC(rc);
6459	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6460	}
6461
6462	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6463	{
6464	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6465	AssertRC(rc);
6466	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6467	}
6468
6469	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6470	{
6471	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6472	AssertRC(rc);
6473	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6474	}
6475	}
6476
6477	/*
6478	* Guest/host EFER MSR.
6479	*/
6480	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6481	{
6482	/* Whether we are using the VMCS to swap the EFER MSR must have been
6483	determined earlier while exporting VM-entry/VM-exit controls. */
6484	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6485	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6486
6487	if (hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
6488	{
6489	/*
6490	* EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
6491	* This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
6492	* VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
6493	* and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
6494	* must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
6495	* the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
6496	* during VM-entry.
6497	*/
6498	uint64_t uGuestEferMsr = pCtx->msrEFER;
6499	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6500	{
6501	if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
6502	uGuestEferMsr &= ~MSR_K6_EFER_LME;
6503	else
6504	Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
6505	}
6506
6507	/*
6508	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6509	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6510	*/
6511	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6512	{
6513	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
6514	AssertRC(rc);
6515	}
6516	else
6517	{
6518	/*
6519	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6520	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6521	*/
6522	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
6523	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6524	AssertRCReturn(rc, rc);
6525	}
6526
6527	Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
6528	}
6529	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6530	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6531
6532	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6533	}
6534
6535	/*
6536	* Other MSRs.
6537	* Speculation Control (R/W).
6538	*/
6539	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6540	{
6541	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6542	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6543	{
6544	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6545	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6546	AssertRCReturn(rc, rc);
6547	}
6548	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6549	}
6550
6551	return VINF_SUCCESS;
6552	}
6553
6554
6555	/**
6556	* Selects up the appropriate function to run guest code.
6557	*
6558	* @returns VBox status code.
6559	* @param pVCpu The cross context virtual CPU structure.
6560	* @param pVmxTransient The VMX-transient structure.
6561	*
6562	* @remarks No-long-jump zone!!!
6563	*/
6564	static int hmR0VmxSelectVMRunHandler(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6565	{
6566	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6567	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6568
6569	if (CPUMIsGuestInLongModeEx(pCtx))
6570	{
6571	#ifndef VBOX_WITH_64_BITS_GUESTS
6572	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6573	#else
6574	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6575	/* Guest is in long mode, use the 64-bit handler (host is 64-bit). */
6576	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6577	#endif
6578	}
6579	else
6580	{
6581	/* Guest is not in long mode, use the 32-bit handler. */
6582	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6583	}
6584	Assert(pVmcsInfo->pfnStartVM);
6585	return VINF_SUCCESS;
6586	}
6587
6588
6589	/**
6590	* Wrapper for running the guest code in VT-x.
6591	*
6592	* @returns VBox status code, no informational status codes.
6593	* @param pVCpu The cross context virtual CPU structure.
6594	* @param pVmxTransient The VMX-transient structure.
6595	*
6596	* @remarks No-long-jump zone!!!
6597	*/
6598	DECLINLINE(int) hmR0VmxRunGuest(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6599	{
6600	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6601	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6602	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6603
6604	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6605
6606	/*
6607	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6608	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6609	* callee-saved and thus the need for this XMM wrapper.
6610	*
6611	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6612	*/
6613	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6614	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6615	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6616	#ifdef VBOX_WITH_KERNEL_USING_XMM
6617	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6618	#else
6619	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu);
6620	#endif
6621	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6622	return rc;
6623	}
6624
6625
6626	/**
6627	* Reports world-switch error and dumps some useful debug info.
6628	*
6629	* @param pVCpu The cross context virtual CPU structure.
6630	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6631	* @param pVmxTransient The VMX-transient structure (only
6632	* exitReason updated).
6633	*/
6634	static void hmR0VmxReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6635	{
6636	Assert(pVCpu);
6637	Assert(pVmxTransient);
6638	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6639
6640	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6641	switch (rcVMRun)
6642	{
6643	case VERR_VMX_INVALID_VMXON_PTR:
6644	AssertFailed();
6645	break;
6646	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6647	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6648	{
6649	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6650	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6651	AssertRC(rc);
6652	hmR0VmxReadExitQualVmcs(pVmxTransient);
6653
6654	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6655	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6656	Cannot do it here as we may have been long preempted. */
6657
6658	#ifdef VBOX_STRICT
6659	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6660	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6661	pVmxTransient->uExitReason));
6662	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6663	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6664	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6665	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6666	else
6667	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6668	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6669	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6670
6671	static struct
6672	{
6673	/** Name of the field to log. */
6674	const char *pszName;
6675	/** The VMCS field. */
6676	uint32_t uVmcsField;
6677	/** Whether host support of this field needs to be checked. */
6678	bool fCheckSupport;
6679	} const s_aVmcsFields[] =
6680	{
6681	{ "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false },
6682	{ "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false },
6683	{ "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true },
6684	{ "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false },
6685	{ "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false },
6686	{ "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false },
6687	{ "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false },
6688	{ "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false },
6689	{ "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false },
6690	{ "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false },
6691	{ "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false },
6692	{ "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false },
6693	{ "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false },
6694	{ "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false },
6695	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false },
6696	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false },
6697	{ "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false },
6698	{ "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false },
6699	{ "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false },
6700	{ "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false },
6701	{ "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true },
6702	{ "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false },
6703	{ "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false },
6704	{ "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false },
6705	{ "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
6706	{ "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false },
6707	{ "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false },
6708	{ "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false },
6709	/* The order of selector fields below are fixed! */
6710	{ "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false },
6711	{ "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false },
6712	{ "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false },
6713	{ "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false },
6714	{ "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false },
6715	{ "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false },
6716	{ "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false },
6717	/* End of ordered selector fields. */
6718	{ "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false },
6719	{ "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false },
6720	{ "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false },
6721	{ "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false },
6722	{ "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false },
6723	{ "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false },
6724	{ "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false },
6725	{ "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false }
6726	};
6727
6728	RTGDTR HostGdtr;
6729	ASMGetGDTR(&HostGdtr);
6730
6731	uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
6732	for (uint32_t i = 0; i < cVmcsFields; i++)
6733	{
6734	uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
6735
6736	bool fSupported;
6737	if (!s_aVmcsFields[i].fCheckSupport)
6738	fSupported = true;
6739	else
6740	{
6741	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6742	switch (uVmcsField)
6743	{
6744	case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hm.s.fNestedPaging; break;
6745	case VMX_VMCS16_VPID: fSupported = pVM->hm.s.vmx.fVpid; break;
6746	case VMX_VMCS32_CTRL_PROC_EXEC2:
6747	fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6748	break;
6749	default:
6750	AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
6751	}
6752	}
6753
6754	if (fSupported)
6755	{
6756	uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
6757	switch (uWidth)
6758	{
6759	case VMX_VMCSFIELD_WIDTH_16BIT:
6760	{
6761	uint16_t u16Val;
6762	rc = VMXReadVmcs16(uVmcsField, &u16Val);
6763	AssertRC(rc);
6764	Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
6765
6766	if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
6767	&& uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
6768	{
6769	if (u16Val < HostGdtr.cbGdt)
6770	{
6771	/* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
6772	static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
6773	"Host FS", "Host GS", "Host TR" };
6774	uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
6775	Assert(idxSel < RT_ELEMENTS(s_apszSel));
6776	PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
6777	hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
6778	}
6779	else
6780	Log4((" Selector value exceeds GDT limit!\n"));
6781	}
6782	break;
6783	}
6784
6785	case VMX_VMCSFIELD_WIDTH_32BIT:
6786	{
6787	uint32_t u32Val;
6788	rc = VMXReadVmcs32(uVmcsField, &u32Val);
6789	AssertRC(rc);
6790	Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
6791	break;
6792	}
6793
6794	case VMX_VMCSFIELD_WIDTH_64BIT:
6795	case VMX_VMCSFIELD_WIDTH_NATURAL:
6796	{
6797	uint64_t u64Val;
6798	rc = VMXReadVmcs64(uVmcsField, &u64Val);
6799	AssertRC(rc);
6800	Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
6801	break;
6802	}
6803	}
6804	}
6805	}
6806
6807	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6808	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6809	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6810	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6811	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6812	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6813	#endif /* VBOX_STRICT */
6814	break;
6815	}
6816
6817	default:
6818	/* Impossible */
6819	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6820	break;
6821	}
6822	}
6823
6824
6825	/**
6826	* Sets up the usage of TSC-offsetting and updates the VMCS.
6827	*
6828	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6829	* VMX-preemption timer.
6830	*
6831	* @returns VBox status code.
6832	* @param pVCpu The cross context virtual CPU structure.
6833	* @param pVmxTransient The VMX-transient structure.
6834	*
6835	* @remarks No-long-jump zone!!!
6836	*/
6837	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6838	{
6839	bool fOffsettedTsc;
6840	bool fParavirtTsc;
6841	uint64_t uTscOffset;
6842	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6843	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6844
6845	if (pVM->hm.s.vmx.fUsePreemptTimer)
6846	{
6847	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6848
6849	/* Make sure the returned values have sane upper and lower boundaries. */
6850	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6851	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6852	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6853	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6854
6855	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6856	* preemption timers here. We probably need to clamp the preemption timer,
6857	* after converting the timer value to the host. */
6858	uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6859	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6860	AssertRC(rc);
6861	}
6862	else
6863	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6864
6865	if (fParavirtTsc)
6866	{
6867	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6868	information before every VM-entry, hence disable it for performance sake. */
6869	#if 0
6870	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6871	AssertRC(rc);
6872	#endif
6873	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6874	}
6875
6876	if ( fOffsettedTsc
6877	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6878	{
6879	if (pVmxTransient->fIsNestedGuest)
6880	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6881	hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
6882	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6883	}
6884	else
6885	{
6886	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6887	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6888	}
6889	}
6890
6891
6892	/**
6893	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6894	* VM-exit interruption info type.
6895	*
6896	* @returns The IEM exception flags.
6897	* @param uVector The event vector.
6898	* @param uVmxEventType The VMX event type.
6899	*
6900	* @remarks This function currently only constructs flags required for
6901	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6902	* and CR2 aspects of an exception are not included).
6903	*/
6904	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6905	{
6906	uint32_t fIemXcptFlags;
6907	switch (uVmxEventType)
6908	{
6909	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6910	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6911	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6912	break;
6913
6914	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6915	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6916	break;
6917
6918	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6919	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6920	break;
6921
6922	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6923	{
6924	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6925	if (uVector == X86_XCPT_BP)
6926	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6927	else if (uVector == X86_XCPT_OF)
6928	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6929	else
6930	{
6931	fIemXcptFlags = 0;
6932	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6933	}
6934	break;
6935	}
6936
6937	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6938	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6939	break;
6940
6941	default:
6942	fIemXcptFlags = 0;
6943	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6944	break;
6945	}
6946	return fIemXcptFlags;
6947	}
6948
6949
6950	/**
6951	* Sets an event as a pending event to be injected into the guest.
6952	*
6953	* @param pVCpu The cross context virtual CPU structure.
6954	* @param u32IntInfo The VM-entry interruption-information field.
6955	* @param cbInstr The VM-entry instruction length in bytes (for
6956	* software interrupts, exceptions and privileged
6957	* software exceptions).
6958	* @param u32ErrCode The VM-entry exception error code.
6959	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6960	* page-fault.
6961	*/
6962	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6963	RTGCUINTPTR GCPtrFaultAddress)
6964	{
6965	Assert(!pVCpu->hm.s.Event.fPending);
6966	pVCpu->hm.s.Event.fPending = true;
6967	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6968	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6969	pVCpu->hm.s.Event.cbInstr = cbInstr;
6970	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6971	}
6972
6973
6974	/**
6975	* Sets an external interrupt as pending-for-injection into the VM.
6976	*
6977	* @param pVCpu The cross context virtual CPU structure.
6978	* @param u8Interrupt The external interrupt vector.
6979	*/
6980	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
6981	{
6982	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6983	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6984	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6985	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6986	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6987	}
6988
6989
6990	/**
6991	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6992	*
6993	* @param pVCpu The cross context virtual CPU structure.
6994	*/
6995	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPUCC pVCpu)
6996	{
6997	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6998	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6999	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7000	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7001	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7002	}
7003
7004
7005	/**
7006	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7007	*
7008	* @param pVCpu The cross context virtual CPU structure.
7009	*/
7010	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPUCC pVCpu)
7011	{
7012	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7013	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7014	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7015	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7016	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7017	}
7018
7019
7020	/**
7021	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7022	*
7023	* @param pVCpu The cross context virtual CPU structure.
7024	*/
7025	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPUCC pVCpu)
7026	{
7027	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7028	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7029	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7030	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7031	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7032	}
7033
7034
7035	/**
7036	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7037	*
7038	* @param pVCpu The cross context virtual CPU structure.
7039	*/
7040	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPUCC pVCpu)
7041	{
7042	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7043	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7044	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7045	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7046	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7047	}
7048
7049
7050	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7051	/**
7052	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7053	*
7054	* @param pVCpu The cross context virtual CPU structure.
7055	* @param u32ErrCode The error code for the general-protection exception.
7056	*/
7057	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7058	{
7059	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7060	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7061	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7062	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7063	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7064	}
7065
7066
7067	/**
7068	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7069	*
7070	* @param pVCpu The cross context virtual CPU structure.
7071	* @param u32ErrCode The error code for the stack exception.
7072	*/
7073	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7074	{
7075	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7076	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7077	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7078	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7079	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7080	}
7081	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7082
7083
7084	/**
7085	* Fixes up attributes for the specified segment register.
7086	*
7087	* @param pVCpu The cross context virtual CPU structure.
7088	* @param pSelReg The segment register that needs fixing.
7089	* @param idxSel The VMCS field for the corresponding segment register.
7090	*/
7091	static void hmR0VmxFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7092	{
7093	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7094
7095	/*
7096	* If VT-x marks the segment as unusable, most other bits remain undefined:
7097	* - For CS the L, D and G bits have meaning.
7098	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7099	* - For the remaining data segments no bits are defined.
7100	*
7101	* The present bit and the unusable bit has been observed to be set at the
7102	* same time (the selector was supposed to be invalid as we started executing
7103	* a V8086 interrupt in ring-0).
7104	*
7105	* What should be important for the rest of the VBox code, is that the P bit is
7106	* cleared. Some of the other VBox code recognizes the unusable bit, but
7107	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7108	* safe side here, we'll strip off P and other bits we don't care about. If
7109	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7110	*
7111	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7112	*/
7113	#ifdef VBOX_STRICT
7114	uint32_t const uAttr = pSelReg->Attr.u;
7115	#endif
7116
7117	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7118	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7119	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7120
7121	#ifdef VBOX_STRICT
7122	VMMRZCallRing3Disable(pVCpu);
7123	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7124	# ifdef DEBUG_bird
7125	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7126	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7127	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7128	# endif
7129	VMMRZCallRing3Enable(pVCpu);
7130	NOREF(uAttr);
7131	#endif
7132	RT_NOREF2(pVCpu, idxSel);
7133	}
7134
7135
7136	/**
7137	* Imports a guest segment register from the current VMCS into the guest-CPU
7138	* context.
7139	*
7140	* @param pVCpu The cross context virtual CPU structure.
7141	* @param iSegReg The segment register number (X86_SREG_XXX).
7142	*
7143	* @remarks Called with interrupts and/or preemption disabled.
7144	*/
7145	static void hmR0VmxImportGuestSegReg(PVMCPUCC pVCpu, uint8_t iSegReg)
7146	{
7147	Assert(iSegReg < X86_SREG_COUNT);
7148
7149	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7150	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7151	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7152	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7153
7154	uint16_t u16Sel;
7155	uint64_t u64Base;
7156	uint32_t u32Limit, u32Attr;
7157	int rc = VMXReadVmcs16(idxSel, &u16Sel); AssertRC(rc);
7158	rc = VMXReadVmcs32(idxLimit, &u32Limit); AssertRC(rc);
7159	rc = VMXReadVmcs32(idxAttr, &u32Attr); AssertRC(rc);
7160	rc = VMXReadVmcsNw(idxBase, &u64Base); AssertRC(rc);
7161
7162	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7163	pSelReg->Sel = u16Sel;
7164	pSelReg->ValidSel = u16Sel;
7165	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7166	pSelReg->u32Limit = u32Limit;
7167	pSelReg->u64Base = u64Base;
7168	pSelReg->Attr.u = u32Attr;
7169	if (u32Attr & X86DESCATTR_UNUSABLE)
7170	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7171	}
7172
7173
7174	/**
7175	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7176	*
7177	* @param pVCpu The cross context virtual CPU structure.
7178	*
7179	* @remarks Called with interrupts and/or preemption disabled.
7180	*/
7181	static void hmR0VmxImportGuestLdtr(PVMCPUCC pVCpu)
7182	{
7183	uint16_t u16Sel;
7184	uint64_t u64Base;
7185	uint32_t u32Limit, u32Attr;
7186	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
7187	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
7188	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7189	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
7190
7191	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
7192	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
7193	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7194	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7195	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7196	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7197	if (u32Attr & X86DESCATTR_UNUSABLE)
7198	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7199	}
7200
7201
7202	/**
7203	* Imports the guest TR from the current VMCS into the guest-CPU context.
7204	*
7205	* @param pVCpu The cross context virtual CPU structure.
7206	*
7207	* @remarks Called with interrupts and/or preemption disabled.
7208	*/
7209	static void hmR0VmxImportGuestTr(PVMCPUCC pVCpu)
7210	{
7211	uint16_t u16Sel;
7212	uint64_t u64Base;
7213	uint32_t u32Limit, u32Attr;
7214	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
7215	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
7216	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7217	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
7218
7219	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
7220	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
7221	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7222	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7223	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7224	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7225	/* TR is the only selector that can never be unusable. */
7226	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7227	}
7228
7229
7230	/**
7231	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7232	*
7233	* @param pVCpu The cross context virtual CPU structure.
7234	*
7235	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7236	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7237	* instead!!!
7238	*/
7239	static void hmR0VmxImportGuestRip(PVMCPUCC pVCpu)
7240	{
7241	uint64_t u64Val;
7242	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7243	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7244	{
7245	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
7246	AssertRC(rc);
7247
7248	pCtx->rip = u64Val;
7249	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7250	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7251	}
7252	}
7253
7254
7255	/**
7256	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7257	*
7258	* @param pVCpu The cross context virtual CPU structure.
7259	* @param pVmcsInfo The VMCS info. object.
7260	*
7261	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7262	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7263	* instead!!!
7264	*/
7265	static void hmR0VmxImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7266	{
7267	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7268	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7269	{
7270	uint64_t u64Val;
7271	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7272	AssertRC(rc);
7273
7274	pCtx->rflags.u64 = u64Val;
7275	if (pVmcsInfo->RealMode.fRealOnV86Active)
7276	{
7277	pCtx->eflags.Bits.u1VM = 0;
7278	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7279	}
7280	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7281	}
7282	}
7283
7284
7285	/**
7286	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7287	* context.
7288	*
7289	* @param pVCpu The cross context virtual CPU structure.
7290	* @param pVmcsInfo The VMCS info. object.
7291	*
7292	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7293	* do not log!
7294	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7295	* instead!!!
7296	*/
7297	static void hmR0VmxImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7298	{
7299	uint32_t u32Val;
7300	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
7301	if (!u32Val)
7302	{
7303	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7304	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7305	CPUMSetGuestNmiBlocking(pVCpu, false);
7306	}
7307	else
7308	{
7309	/*
7310	* We must import RIP here to set our EM interrupt-inhibited state.
7311	* We also import RFLAGS as our code that evaluates pending interrupts
7312	* before VM-entry requires it.
7313	*/
7314	hmR0VmxImportGuestRip(pVCpu);
7315	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7316
7317	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7318	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7319	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7320	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7321
7322	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7323	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7324	}
7325	}
7326
7327
7328	/**
7329	* Worker for VMXR0ImportStateOnDemand.
7330	*
7331	* @returns VBox status code.
7332	* @param pVCpu The cross context virtual CPU structure.
7333	* @param pVmcsInfo The VMCS info. object.
7334	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7335	*/
7336	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7337	{
7338	int rc = VINF_SUCCESS;
7339	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7340	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7341	uint32_t u32Val;
7342
7343	/*
7344	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7345	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7346	* neither are other host platforms.
7347	*
7348	* Committing this temporarily as it prevents BSOD.
7349	*
7350	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7351	*/
7352	#ifdef RT_OS_WINDOWS
7353	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7354	return VERR_HM_IPE_1;
7355	#endif
7356
7357	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7358
7359	/*
7360	* We disable interrupts to make the updating of the state and in particular
7361	* the fExtrn modification atomic wrt to preemption hooks.
7362	*/
7363	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7364
7365	fWhat &= pCtx->fExtrn;
7366	if (fWhat)
7367	{
7368	do
7369	{
7370	if (fWhat & CPUMCTX_EXTRN_RIP)
7371	hmR0VmxImportGuestRip(pVCpu);
7372
7373	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7374	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7375
7376	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7377	hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7378
7379	if (fWhat & CPUMCTX_EXTRN_RSP)
7380	{
7381	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &pCtx->rsp);
7382	AssertRC(rc);
7383	}
7384
7385	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7386	{
7387	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7388	if (fWhat & CPUMCTX_EXTRN_CS)
7389	{
7390	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7391	hmR0VmxImportGuestRip(pVCpu);
7392	if (fRealOnV86Active)
7393	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7394	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7395	}
7396	if (fWhat & CPUMCTX_EXTRN_SS)
7397	{
7398	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7399	if (fRealOnV86Active)
7400	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7401	}
7402	if (fWhat & CPUMCTX_EXTRN_DS)
7403	{
7404	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7405	if (fRealOnV86Active)
7406	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7407	}
7408	if (fWhat & CPUMCTX_EXTRN_ES)
7409	{
7410	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7411	if (fRealOnV86Active)
7412	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7413	}
7414	if (fWhat & CPUMCTX_EXTRN_FS)
7415	{
7416	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7417	if (fRealOnV86Active)
7418	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7419	}
7420	if (fWhat & CPUMCTX_EXTRN_GS)
7421	{
7422	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7423	if (fRealOnV86Active)
7424	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7425	}
7426	}
7427
7428	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7429	{
7430	if (fWhat & CPUMCTX_EXTRN_LDTR)
7431	hmR0VmxImportGuestLdtr(pVCpu);
7432
7433	if (fWhat & CPUMCTX_EXTRN_GDTR)
7434	{
7435	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
7436	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
7437	pCtx->gdtr.cbGdt = u32Val;
7438	}
7439
7440	/* Guest IDTR. */
7441	if (fWhat & CPUMCTX_EXTRN_IDTR)
7442	{
7443	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
7444	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
7445	pCtx->idtr.cbIdt = u32Val;
7446	}
7447
7448	/* Guest TR. */
7449	if (fWhat & CPUMCTX_EXTRN_TR)
7450	{
7451	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7452	don't need to import that one. */
7453	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7454	hmR0VmxImportGuestTr(pVCpu);
7455	}
7456	}
7457
7458	if (fWhat & CPUMCTX_EXTRN_DR7)
7459	{
7460	if (!pVCpu->hm.s.fUsingHyperDR7)
7461	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &pCtx->dr[7]); AssertRC(rc);
7462	}
7463
7464	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7465	{
7466	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
7467	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
7468	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
7469	pCtx->SysEnter.cs = u32Val;
7470	}
7471
7472	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7473	{
7474	if ( pVM->hm.s.fAllow64BitGuests
7475	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7476	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7477	}
7478
7479	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7480	{
7481	if ( pVM->hm.s.fAllow64BitGuests
7482	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7483	{
7484	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7485	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7486	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7487	}
7488	}
7489
7490	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7491	{
7492	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7493	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7494	Assert(pMsrs);
7495	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7496	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7497	for (uint32_t i = 0; i < cMsrs; i++)
7498	{
7499	uint32_t const idMsr = pMsrs[i].u32Msr;
7500	switch (idMsr)
7501	{
7502	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7503	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7504	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7505	default:
7506	{
7507	pCtx->fExtrn = 0;
7508	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7509	ASMSetFlags(fEFlags);
7510	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7511	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7512	}
7513	}
7514	}
7515	}
7516
7517	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7518	{
7519	if (fWhat & CPUMCTX_EXTRN_CR0)
7520	{
7521	uint64_t u64Cr0;
7522	uint64_t u64Shadow;
7523	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
7524	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
7525	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7526	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7527	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7528	#else
7529	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7530	{
7531	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7532	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7533	}
7534	else
7535	{
7536	/*
7537	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
7538	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7539	* re-construct CR0. See @bugref{9180#c95} for details.
7540	*/
7541	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7542	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7543	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7544	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
7545	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
7546	}
7547	#endif
7548	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7549	CPUMSetGuestCR0(pVCpu, u64Cr0);
7550	VMMRZCallRing3Enable(pVCpu);
7551	}
7552
7553	if (fWhat & CPUMCTX_EXTRN_CR4)
7554	{
7555	uint64_t u64Cr4;
7556	uint64_t u64Shadow;
7557	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
7558	rc \|= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
7559	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7560	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7561	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7562	#else
7563	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7564	{
7565	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7566	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7567	}
7568	else
7569	{
7570	/*
7571	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
7572	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7573	* re-construct CR4. See @bugref{9180#c95} for details.
7574	*/
7575	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7576	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7577	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7578	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
7579	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
7580	}
7581	#endif
7582	pCtx->cr4 = u64Cr4;
7583	}
7584
7585	if (fWhat & CPUMCTX_EXTRN_CR3)
7586	{
7587	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7588	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7589	\|\| ( pVM->hm.s.fNestedPaging
7590	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7591	{
7592	uint64_t u64Cr3;
7593	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
7594	if (pCtx->cr3 != u64Cr3)
7595	{
7596	pCtx->cr3 = u64Cr3;
7597	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7598	}
7599
7600	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7601	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7602	if (CPUMIsGuestInPAEModeEx(pCtx))
7603	{
7604	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
7605	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
7606	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
7607	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
7608	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7609	}
7610	}
7611	}
7612	}
7613
7614	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7615	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7616	{
7617	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7618	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7619	{
7620	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7621	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7622	if (RT_SUCCESS(rc))
7623	{ /* likely */ }
7624	else
7625	break;
7626	}
7627	}
7628	#endif
7629	} while (0);
7630
7631	if (RT_SUCCESS(rc))
7632	{
7633	/* Update fExtrn. */
7634	pCtx->fExtrn &= ~fWhat;
7635
7636	/* If everything has been imported, clear the HM keeper bit. */
7637	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7638	{
7639	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7640	Assert(!pCtx->fExtrn);
7641	}
7642	}
7643	}
7644	else
7645	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7646
7647	/*
7648	* Restore interrupts.
7649	*/
7650	ASMSetFlags(fEFlags);
7651
7652	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7653
7654	if (RT_SUCCESS(rc))
7655	{ /* likely */ }
7656	else
7657	return rc;
7658
7659	/*
7660	* Honor any pending CR3 updates.
7661	*
7662	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
7663	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7664	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7665	*
7666	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7667	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7668	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7669	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7670	*
7671	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7672	*/
7673	if (VMMRZCallRing3IsEnabled(pVCpu))
7674	{
7675	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7676	{
7677	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7678	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7679	}
7680
7681	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7682	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7683
7684	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7685	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7686	}
7687
7688	return VINF_SUCCESS;
7689	}
7690
7691
7692	/**
7693	* Saves the guest state from the VMCS into the guest-CPU context.
7694	*
7695	* @returns VBox status code.
7696	* @param pVCpu The cross context virtual CPU structure.
7697	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7698	*/
7699	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
7700	{
7701	AssertPtr(pVCpu);
7702	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7703	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7704	}
7705
7706
7707	/**
7708	* Check per-VM and per-VCPU force flag actions that require us to go back to
7709	* ring-3 for one reason or another.
7710	*
7711	* @returns Strict VBox status code (i.e. informational status codes too)
7712	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7713	* ring-3.
7714	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7715	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7716	* interrupts)
7717	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7718	* all EMTs to be in ring-3.
7719	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7720	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7721	* to the EM loop.
7722	*
7723	* @param pVCpu The cross context virtual CPU structure.
7724	* @param fStepping Whether we are single-stepping the guest using the
7725	* hypervisor debugger.
7726	*
7727	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7728	* is no longer in VMX non-root mode.
7729	*/
7730	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPUCC pVCpu, bool fStepping)
7731	{
7732	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7733
7734	/*
7735	* Update pending interrupts into the APIC's IRR.
7736	*/
7737	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7738	APICUpdatePendingInterrupts(pVCpu);
7739
7740	/*
7741	* Anything pending? Should be more likely than not if we're doing a good job.
7742	*/
7743	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7744	if ( !fStepping
7745	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7746	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7747	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7748	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7749	return VINF_SUCCESS;
7750
7751	/* Pending PGM C3 sync. */
7752	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7753	{
7754	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7755	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7756	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7757	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7758	if (rcStrict != VINF_SUCCESS)
7759	{
7760	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7761	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7762	return rcStrict;
7763	}
7764	}
7765
7766	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7767	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7768	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7769	{
7770	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7771	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7772	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7773	return rc;
7774	}
7775
7776	/* Pending VM request packets, such as hardware interrupts. */
7777	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7778	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7779	{
7780	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7781	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7782	return VINF_EM_PENDING_REQUEST;
7783	}
7784
7785	/* Pending PGM pool flushes. */
7786	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7787	{
7788	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7789	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7790	return VINF_PGM_POOL_FLUSH_PENDING;
7791	}
7792
7793	/* Pending DMA requests. */
7794	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7795	{
7796	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7797	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7798	return VINF_EM_RAW_TO_R3;
7799	}
7800
7801	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7802	/* Pending nested-guest APIC-write (has highest priority among nested-guest FFs). */
7803	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7804	{
7805	Log4Func(("Pending nested-guest APIC-write\n"));
7806	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
7807	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7808	return rcStrict;
7809	}
7810	/** @todo VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_PREEMPT_TIMER */
7811	#endif
7812
7813	return VINF_SUCCESS;
7814	}
7815
7816
7817	/**
7818	* Converts any TRPM trap into a pending HM event. This is typically used when
7819	* entering from ring-3 (not longjmp returns).
7820	*
7821	* @param pVCpu The cross context virtual CPU structure.
7822	*/
7823	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
7824	{
7825	Assert(TRPMHasTrap(pVCpu));
7826	Assert(!pVCpu->hm.s.Event.fPending);
7827
7828	uint8_t uVector;
7829	TRPMEVENT enmTrpmEvent;
7830	uint32_t uErrCode;
7831	RTGCUINTPTR GCPtrFaultAddress;
7832	uint8_t cbInstr;
7833	bool fIcebp;
7834
7835	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
7836	AssertRC(rc);
7837
7838	uint32_t u32IntInfo;
7839	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
7840	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
7841
7842	rc = TRPMResetTrap(pVCpu);
7843	AssertRC(rc);
7844	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7845	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7846
7847	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7848	}
7849
7850
7851	/**
7852	* Converts the pending HM event into a TRPM trap.
7853	*
7854	* @param pVCpu The cross context virtual CPU structure.
7855	*/
7856	static void hmR0VmxPendingEventToTrpmTrap(PVMCPUCC pVCpu)
7857	{
7858	Assert(pVCpu->hm.s.Event.fPending);
7859
7860	/* If a trap was already pending, we did something wrong! */
7861	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7862
7863	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
7864	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
7865	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
7866
7867	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7868
7869	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7870	AssertRC(rc);
7871
7872	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7873	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
7874
7875	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
7876	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7877	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
7878	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7879
7880	if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7881	TRPMSetTrapDueToIcebp(pVCpu);
7882
7883	/* We're now done converting the pending event. */
7884	pVCpu->hm.s.Event.fPending = false;
7885	}
7886
7887
7888	/**
7889	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7890	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7891	*
7892	* @param pVCpu The cross context virtual CPU structure.
7893	* @param pVmcsInfo The VMCS info. object.
7894	*/
7895	static void hmR0VmxSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7896	{
7897	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7898	{
7899	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7900	{
7901	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7902	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7903	AssertRC(rc);
7904	}
7905	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7906	}
7907
7908
7909	/**
7910	* Clears the interrupt-window exiting control in the VMCS.
7911	*
7912	* @param pVmcsInfo The VMCS info. object.
7913	*/
7914	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7915	{
7916	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7917	{
7918	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7919	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7920	AssertRC(rc);
7921	}
7922	}
7923
7924
7925	/**
7926	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7927	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7928	*
7929	* @param pVCpu The cross context virtual CPU structure.
7930	* @param pVmcsInfo The VMCS info. object.
7931	*/
7932	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7933	{
7934	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7935	{
7936	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7937	{
7938	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7939	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7940	AssertRC(rc);
7941	Log4Func(("Setup NMI-window exiting\n"));
7942	}
7943	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7944	}
7945
7946
7947	/**
7948	* Clears the NMI-window exiting control in the VMCS.
7949	*
7950	* @param pVmcsInfo The VMCS info. object.
7951	*/
7952	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7953	{
7954	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7955	{
7956	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7957	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7958	AssertRC(rc);
7959	}
7960	}
7961
7962
7963	/**
7964	* Does the necessary state syncing before returning to ring-3 for any reason
7965	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7966	*
7967	* @returns VBox status code.
7968	* @param pVCpu The cross context virtual CPU structure.
7969	* @param fImportState Whether to import the guest state from the VMCS back
7970	* to the guest-CPU context.
7971	*
7972	* @remarks No-long-jmp zone!!!
7973	*/
7974	static int hmR0VmxLeave(PVMCPUCC pVCpu, bool fImportState)
7975	{
7976	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7977	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7978
7979	RTCPUID const idCpu = RTMpCpuId();
7980	Log4Func(("HostCpuId=%u\n", idCpu));
7981
7982	/*
7983	* !!! IMPORTANT !!!
7984	* If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
7985	*/
7986
7987	/* Save the guest state if necessary. */
7988	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7989	if (fImportState)
7990	{
7991	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
7992	AssertRCReturn(rc, rc);
7993	}
7994
7995	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7996	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7997	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7998
7999	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8000	#ifdef VBOX_STRICT
8001	if (CPUMIsHyperDebugStateActive(pVCpu))
8002	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8003	#endif
8004	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8005	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8006	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8007
8008	/* Restore host-state bits that VT-x only restores partially. */
8009	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8010	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8011	{
8012	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8013	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8014	}
8015	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8016
8017	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8018	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8019	{
8020	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8021	if (!fImportState)
8022	{
8023	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8024	AssertRCReturn(rc, rc);
8025	}
8026	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8027	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8028	}
8029	else
8030	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8031
8032	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8033	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8034
8035	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8036	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8037	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8038	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8039	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8040	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8041	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8042	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8043	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8044	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8045
8046	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8047
8048	/** @todo This partially defeats the purpose of having preemption hooks.
8049	* The problem is, deregistering the hooks should be moved to a place that
8050	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8051	* context.
8052	*/
8053	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8054	AssertRCReturn(rc, rc);
8055
8056	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8057	/*
8058	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8059	* clear a shadow VMCS before allowing that VMCS to become active on another
8060	* logical processor. We may or may not be importing guest state which clears
8061	* it, so cover for it here.
8062	*
8063	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8064	*/
8065	if ( pVmcsInfo->pvShadowVmcs
8066	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8067	{
8068	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8069	AssertRCReturn(rc, rc);
8070	}
8071
8072	/*
8073	* Flag that we need to re-export the host state if we switch to this VMCS before
8074	* executing guest or nested-guest code.
8075	*/
8076	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
8077	#endif
8078
8079	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8080	NOREF(idCpu);
8081	return VINF_SUCCESS;
8082	}
8083
8084
8085	/**
8086	* Leaves the VT-x session.
8087	*
8088	* @returns VBox status code.
8089	* @param pVCpu The cross context virtual CPU structure.
8090	*
8091	* @remarks No-long-jmp zone!!!
8092	*/
8093	static int hmR0VmxLeaveSession(PVMCPUCC pVCpu)
8094	{
8095	HM_DISABLE_PREEMPT(pVCpu);
8096	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8097	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8098	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8099
8100	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8101	and done this from the VMXR0ThreadCtxCallback(). */
8102	if (!pVCpu->hm.s.fLeaveDone)
8103	{
8104	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8105	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8106	pVCpu->hm.s.fLeaveDone = true;
8107	}
8108	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8109
8110	/*
8111	* !!! IMPORTANT !!!
8112	* If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
8113	*/
8114
8115	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8116	/** @todo Deregistering here means we need to VMCLEAR always
8117	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8118	* for calling VMMR0ThreadCtxHookDisable here! */
8119	VMMR0ThreadCtxHookDisable(pVCpu);
8120
8121	/* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
8122	int rc = HMR0LeaveCpu(pVCpu);
8123	HM_RESTORE_PREEMPT();
8124	return rc;
8125	}
8126
8127
8128	/**
8129	* Does the necessary state syncing before doing a longjmp to ring-3.
8130	*
8131	* @returns VBox status code.
8132	* @param pVCpu The cross context virtual CPU structure.
8133	*
8134	* @remarks No-long-jmp zone!!!
8135	*/
8136	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPUCC pVCpu)
8137	{
8138	return hmR0VmxLeaveSession(pVCpu);
8139	}
8140
8141
8142	/**
8143	* Take necessary actions before going back to ring-3.
8144	*
8145	* An action requires us to go back to ring-3. This function does the necessary
8146	* steps before we can safely return to ring-3. This is not the same as longjmps
8147	* to ring-3, this is voluntary and prepares the guest so it may continue
8148	* executing outside HM (recompiler/IEM).
8149	*
8150	* @returns VBox status code.
8151	* @param pVCpu The cross context virtual CPU structure.
8152	* @param rcExit The reason for exiting to ring-3. Can be
8153	* VINF_VMM_UNKNOWN_RING3_CALL.
8154	*/
8155	static int hmR0VmxExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
8156	{
8157	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8158
8159	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8160	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8161	{
8162	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8163	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8164	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8165	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8166	}
8167
8168	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8169	VMMRZCallRing3Disable(pVCpu);
8170	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8171
8172	/*
8173	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8174	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8175	*
8176	* This is because execution may continue from ring-3 and we would need to inject
8177	* the event from there (hence place it back in TRPM).
8178	*/
8179	if (pVCpu->hm.s.Event.fPending)
8180	{
8181	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8182	Assert(!pVCpu->hm.s.Event.fPending);
8183
8184	/* Clear the events from the VMCS. */
8185	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8186	AssertRC(rc);
8187	}
8188	#ifdef VBOX_STRICT
8189	else
8190	{
8191	/*
8192	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8193	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8194	* occasionally, see @bugref{9180#c42}.
8195	*
8196	* However, if the VM-entry failed, any VM entry-interruption info. field would
8197	* be left unmodified as the event would not have been injected to the guest. In
8198	* such cases, don't assert, we're not going to continue guest execution anyway.
8199	*/
8200	uint32_t uExitReason;
8201	uint32_t uEntryIntInfo;
8202	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8203	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8204	AssertRC(rc);
8205	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8206	}
8207	#endif
8208
8209	/*
8210	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8211	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8212	* (e.g. TPR below threshold).
8213	*/
8214	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8215	{
8216	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8217	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8218	}
8219
8220	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8221	and if we're injecting an event we should have a TRPM trap pending. */
8222	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8223	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8224	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8225	#endif
8226
8227	/* Save guest state and restore host state bits. */
8228	int rc = hmR0VmxLeaveSession(pVCpu);
8229	AssertRCReturn(rc, rc);
8230	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8231
8232	/* Thread-context hooks are unregistered at this point!!! */
8233	/* Ring-3 callback notifications are unregistered at this point!!! */
8234
8235	/* Sync recompiler state. */
8236	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8237	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8238	\| CPUM_CHANGED_LDTR
8239	\| CPUM_CHANGED_GDTR
8240	\| CPUM_CHANGED_IDTR
8241	\| CPUM_CHANGED_TR
8242	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8243	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8244	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8245	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8246
8247	Assert(!pVCpu->hm.s.fClearTrapFlag);
8248
8249	/* Update the exit-to-ring 3 reason. */
8250	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8251
8252	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8253	if ( rcExit != VINF_EM_RAW_INTERRUPT
8254	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8255	{
8256	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8257	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8258	}
8259
8260	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8261	VMMRZCallRing3Enable(pVCpu);
8262	return rc;
8263	}
8264
8265
8266	/**
8267	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8268	* longjump to ring-3 and possibly get preempted.
8269	*
8270	* @returns VBox status code.
8271	* @param pVCpu The cross context virtual CPU structure.
8272	* @param enmOperation The operation causing the ring-3 longjump.
8273	*/
8274	VMMR0DECL(int) VMXR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
8275	{
8276	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8277	{
8278	/*
8279	* !!! IMPORTANT !!!
8280	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8281	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8282	*/
8283	VMMRZCallRing3RemoveNotification(pVCpu);
8284	VMMRZCallRing3Disable(pVCpu);
8285	HM_DISABLE_PREEMPT(pVCpu);
8286
8287	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8288	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8289	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8290	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8291
8292	/* Restore host-state bits that VT-x only restores partially. */
8293	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8294	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8295	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8296	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8297
8298	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8299	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8300	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8301
8302	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8303	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8304	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8305
8306	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8307	cleared as part of importing the guest state above. */
8308	hmR0VmxClearVmcs(pVmcsInfo);
8309
8310	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8311	VMMR0ThreadCtxHookDisable(pVCpu);
8312
8313	/* Leave HM context. This takes care of local init (term). */
8314	HMR0LeaveCpu(pVCpu);
8315	HM_RESTORE_PREEMPT();
8316	return VINF_SUCCESS;
8317	}
8318
8319	Assert(pVCpu);
8320	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8321	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8322
8323	VMMRZCallRing3Disable(pVCpu);
8324	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8325
8326	Log4Func(("-> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8327
8328	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8329	AssertRCReturn(rc, rc);
8330
8331	VMMRZCallRing3Enable(pVCpu);
8332	return VINF_SUCCESS;
8333	}
8334
8335
8336	/**
8337	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8338	* stack.
8339	*
8340	* @returns Strict VBox status code (i.e. informational status codes too).
8341	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8342	* @param pVCpu The cross context virtual CPU structure.
8343	* @param uValue The value to push to the guest stack.
8344	*/
8345	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
8346	{
8347	/*
8348	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8349	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8350	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8351	*/
8352	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8353	if (pCtx->sp == 1)
8354	return VINF_EM_RESET;
8355	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8356	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8357	AssertRC(rc);
8358	return rc;
8359	}
8360
8361
8362	/**
8363	* Injects an event into the guest upon VM-entry by updating the relevant fields
8364	* in the VM-entry area in the VMCS.
8365	*
8366	* @returns Strict VBox status code (i.e. informational status codes too).
8367	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8368	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8369	*
8370	* @param pVCpu The cross context virtual CPU structure.
8371	* @param pVmxTransient The VMX-transient structure.
8372	* @param pEvent The event being injected.
8373	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8374	* will be updated if necessary. This cannot not be NULL.
8375	* @param fStepping Whether we're single-stepping guest execution and should
8376	* return VINF_EM_DBG_STEPPED if the event is injected
8377	* directly (registers modified by us, not by hardware on
8378	* VM-entry).
8379	*/
8380	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8381	uint32_t *pfIntrState)
8382	{
8383	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8384	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8385	Assert(pfIntrState);
8386
8387	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8388	uint32_t u32IntInfo = pEvent->u64IntInfo;
8389	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8390	uint32_t const cbInstr = pEvent->cbInstr;
8391	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8392	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8393	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8394
8395	#ifdef VBOX_STRICT
8396	/*
8397	* Validate the error-code-valid bit for hardware exceptions.
8398	* No error codes for exceptions in real-mode.
8399	*
8400	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8401	*/
8402	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8403	&& !CPUMIsGuestInRealModeEx(pCtx))
8404	{
8405	switch (uVector)
8406	{
8407	case X86_XCPT_PF:
8408	case X86_XCPT_DF:
8409	case X86_XCPT_TS:
8410	case X86_XCPT_NP:
8411	case X86_XCPT_SS:
8412	case X86_XCPT_GP:
8413	case X86_XCPT_AC:
8414	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8415	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8416	RT_FALL_THRU();
8417	default:
8418	break;
8419	}
8420	}
8421
8422	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8423	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8424	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8425	#endif
8426
8427	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8428
8429	/*
8430	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8431	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8432	* interrupt handler in the (real-mode) guest.
8433	*
8434	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8435	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8436	*/
8437	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8438	{
8439	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8440	{
8441	/*
8442	* For CPUs with unrestricted guest execution enabled and with the guest
8443	* in real-mode, we must not set the deliver-error-code bit.
8444	*
8445	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8446	*/
8447	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8448	}
8449	else
8450	{
8451	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8452	Assert(PDMVmmDevHeapIsEnabled(pVM));
8453	Assert(pVM->hm.s.vmx.pRealModeTSS);
8454	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8455
8456	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8457	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8458	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8459	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8460	AssertRCReturn(rc2, rc2);
8461
8462	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8463	size_t const cbIdtEntry = sizeof(X86IDTR16);
8464	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8465	{
8466	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8467	if (uVector == X86_XCPT_DF)
8468	return VINF_EM_RESET;
8469
8470	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8471	No error codes for exceptions in real-mode. */
8472	if (uVector == X86_XCPT_GP)
8473	{
8474	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8475	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8476	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8477	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8478	HMEVENT EventXcptDf;
8479	RT_ZERO(EventXcptDf);
8480	EventXcptDf.u64IntInfo = uXcptDfInfo;
8481	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8482	}
8483
8484	/*
8485	* If we're injecting an event with no valid IDT entry, inject a #GP.
8486	* No error codes for exceptions in real-mode.
8487	*
8488	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8489	*/
8490	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8491	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8492	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8493	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8494	HMEVENT EventXcptGp;
8495	RT_ZERO(EventXcptGp);
8496	EventXcptGp.u64IntInfo = uXcptGpInfo;
8497	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8498	}
8499
8500	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8501	uint16_t uGuestIp = pCtx->ip;
8502	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8503	{
8504	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8505	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8506	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8507	}
8508	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8509	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8510
8511	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8512	X86IDTR16 IdtEntry;
8513	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8514	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8515	AssertRCReturn(rc2, rc2);
8516
8517	/* Construct the stack frame for the interrupt/exception handler. */
8518	VBOXSTRICTRC rcStrict;
8519	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8520	if (rcStrict == VINF_SUCCESS)
8521	{
8522	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8523	if (rcStrict == VINF_SUCCESS)
8524	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8525	}
8526
8527	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8528	if (rcStrict == VINF_SUCCESS)
8529	{
8530	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8531	pCtx->rip = IdtEntry.offSel;
8532	pCtx->cs.Sel = IdtEntry.uSel;
8533	pCtx->cs.ValidSel = IdtEntry.uSel;
8534	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8535	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8536	&& uVector == X86_XCPT_PF)
8537	pCtx->cr2 = GCPtrFault;
8538
8539	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8540	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8541	\| HM_CHANGED_GUEST_RSP);
8542
8543	/*
8544	* If we delivered a hardware exception (other than an NMI) and if there was
8545	* block-by-STI in effect, we should clear it.
8546	*/
8547	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8548	{
8549	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8550	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8551	Log4Func(("Clearing inhibition due to STI\n"));
8552	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8553	}
8554
8555	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8556	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8557
8558	/*
8559	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8560	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8561	*/
8562	pVCpu->hm.s.Event.fPending = false;
8563
8564	/*
8565	* If we eventually support nested-guest execution without unrestricted guest execution,
8566	* we should set fInterceptEvents here.
8567	*/
8568	Assert(!pVmxTransient->fIsNestedGuest);
8569
8570	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8571	if (fStepping)
8572	rcStrict = VINF_EM_DBG_STEPPED;
8573	}
8574	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8575	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8576	return rcStrict;
8577	}
8578	}
8579
8580	/*
8581	* Validate.
8582	*/
8583	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8584	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8585
8586	/*
8587	* Inject the event into the VMCS.
8588	*/
8589	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8590	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8591	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8592	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8593	AssertRC(rc);
8594
8595	/*
8596	* Update guest CR2 if this is a page-fault.
8597	*/
8598	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8599	pCtx->cr2 = GCPtrFault;
8600
8601	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8602	return VINF_SUCCESS;
8603	}
8604
8605
8606	/**
8607	* Evaluates the event to be delivered to the guest and sets it as the pending
8608	* event.
8609	*
8610	* @returns Strict VBox status code (i.e. informational status codes too).
8611	* @param pVCpu The cross context virtual CPU structure.
8612	* @param pVmxTransient The VMX-transient structure.
8613	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8614	*/
8615	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8616	{
8617	Assert(pfIntrState);
8618	Assert(!TRPMHasTrap(pVCpu));
8619
8620	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8621	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8622	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8623
8624	/*
8625	* Get the current interruptibility-state of the guest or nested-guest and
8626	* then figure out what needs to be injected.
8627	*/
8628	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8629	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8630	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8631	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8632
8633	/* We don't support block-by-SMI yet.*/
8634	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
8635
8636	/* Block-by-STI must not be set when interrupts are disabled. */
8637	if (fBlockSti)
8638	{
8639	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
8640	Assert(pCtx->eflags.Bits.u1IF);
8641	}
8642
8643	/* Update interruptibility state to the caller. */
8644	*pfIntrState = fIntrState;
8645
8646	/*
8647	* Toggling of interrupt force-flags here is safe since we update TRPM on
8648	* premature exits to ring-3 before executing guest code, see hmR0VmxExitToRing3().
8649	* We must NOT restore these force-flags.
8650	*/
8651
8652	/** @todo SMI. SMIs take priority over NMIs. */
8653
8654	/*
8655	* Check if an NMI is pending and if the guest or nested-guest can receive them.
8656	* NMIs take priority over external interrupts.
8657	*/
8658	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8659	{
8660	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8661	if ( !pVCpu->hm.s.Event.fPending
8662	&& !fBlockNmi
8663	&& !fBlockSti
8664	&& !fBlockMovSS)
8665	{
8666	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8667	if ( fIsNestedGuest
8668	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8669	return IEMExecVmxVmexitXcptNmi(pVCpu);
8670	#endif
8671	hmR0VmxSetPendingXcptNmi(pVCpu);
8672	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8673	Log4Func(("Pending NMI\n"));
8674	}
8675	else if (!fIsNestedGuest)
8676	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8677	/* else: for nested-guests, NMI-window exiting will be picked up when merging VMCS controls. */
8678	}
8679	/*
8680	* Check if an external interrupt (PIC/APIC) is pending and if the guest or nested-guest
8681	* can receive them. Once PDMGetInterrupt() returns a valid interrupt we -must- deliver
8682	* the interrupt. We can no longer re-request it from the APIC.
8683	*/
8684	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8685	&& !pVCpu->hm.s.fSingleInstruction)
8686	{
8687	Assert(!DBGFIsStepping(pVCpu));
8688	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8689	AssertRCReturn(rc, rc);
8690
8691	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8692	if ( !pVCpu->hm.s.Event.fPending
8693	&& !fBlockInt
8694	&& !fBlockSti
8695	&& !fBlockMovSS)
8696	{
8697	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8698	if ( fIsNestedGuest
8699	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8700	&& !CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8701	{
8702	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8703	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8704	return rcStrict;
8705	}
8706	#endif
8707	uint8_t u8Interrupt;
8708	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8709	if (RT_SUCCESS(rc))
8710	{
8711	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8712	if ( fIsNestedGuest
8713	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8714	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8715	{
8716	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8717	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8718	return rcStrict;
8719	}
8720	#endif
8721	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8722	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8723	}
8724	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8725	{
8726	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8727
8728	if ( !fIsNestedGuest
8729	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8730	hmR0VmxApicSetTprThreshold(pVmcsInfo, u8Interrupt >> 4);
8731	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8732
8733	/*
8734	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8735	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8736	* need to re-set this force-flag here.
8737	*/
8738	}
8739	else
8740	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8741	}
8742	else if (!fIsNestedGuest)
8743	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8744	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCS controls. */
8745	}
8746
8747	return VINF_SUCCESS;
8748	}
8749
8750
8751	/**
8752	* Injects any pending events into the guest if the guest is in a state to
8753	* receive them.
8754	*
8755	* @returns Strict VBox status code (i.e. informational status codes too).
8756	* @param pVCpu The cross context virtual CPU structure.
8757	* @param pVmxTransient The VMX-transient structure.
8758	* @param fIntrState The VT-x guest-interruptibility state.
8759	* @param fStepping Whether we are single-stepping the guest using the
8760	* hypervisor debugger and should return
8761	* VINF_EM_DBG_STEPPED if the event was dispatched
8762	* directly.
8763	*/
8764	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8765	{
8766	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8767	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8768
8769	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8770	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8771
8772	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8773	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8774	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8775	Assert(!TRPMHasTrap(pVCpu));
8776
8777	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8778	if (pVCpu->hm.s.Event.fPending)
8779	{
8780	/*
8781	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8782	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8783	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8784	*
8785	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8786	*/
8787	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8788	#ifdef VBOX_STRICT
8789	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8790	{
8791	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8792	Assert(!fBlockInt);
8793	Assert(!fBlockSti);
8794	Assert(!fBlockMovSS);
8795	}
8796	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8797	{
8798	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8799	Assert(!fBlockSti);
8800	Assert(!fBlockMovSS);
8801	Assert(!fBlockNmi);
8802	}
8803	#endif
8804	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8805	uIntType));
8806
8807	/*
8808	* Inject the event and get any changes to the guest-interruptibility state.
8809	*
8810	* The guest-interruptibility state may need to be updated if we inject the event
8811	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8812	*/
8813	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8814	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8815
8816	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8817	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8818	else
8819	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8820	}
8821
8822	/*
8823	* Update the guest-interruptibility state.
8824	*
8825	* This is required for the real-on-v86 software interrupt injection case above, as well as
8826	* updates to the guest state from ring-3 or IEM/REM.
8827	*/
8828	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8829	AssertRC(rc);
8830
8831	/*
8832	* There's no need to clear the VM-entry interruption-information field here if we're not
8833	* injecting anything. VT-x clears the valid bit on every VM-exit.
8834	*
8835	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8836	*/
8837
8838	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8839	NOREF(fBlockMovSS); NOREF(fBlockSti);
8840	return rcStrict;
8841	}
8842
8843
8844	/**
8845	* Enters the VT-x session.
8846	*
8847	* @returns VBox status code.
8848	* @param pVCpu The cross context virtual CPU structure.
8849	*/
8850	VMMR0DECL(int) VMXR0Enter(PVMCPUCC pVCpu)
8851	{
8852	AssertPtr(pVCpu);
8853	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8854	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8855
8856	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8857	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8858	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8859
8860	#ifdef VBOX_STRICT
8861	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8862	RTCCUINTREG uHostCr4 = ASMGetCR4();
8863	if (!(uHostCr4 & X86_CR4_VMXE))
8864	{
8865	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8866	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8867	}
8868	#endif
8869
8870	/*
8871	* Load the appropriate VMCS as the current and active one.
8872	*/
8873	PVMXVMCSINFO pVmcsInfo;
8874	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8875	if (!fInNestedGuestMode)
8876	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8877	else
8878	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8879	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8880	if (RT_SUCCESS(rc))
8881	{
8882	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8883	pVCpu->hm.s.fLeaveDone = false;
8884	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8885
8886	/*
8887	* Do the EMT scheduled L1D flush here if needed.
8888	*/
8889	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8890	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8891	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8892	hmR0MdsClear();
8893	}
8894	return rc;
8895	}
8896
8897
8898	/**
8899	* The thread-context callback (only on platforms which support it).
8900	*
8901	* @param enmEvent The thread-context event.
8902	* @param pVCpu The cross context virtual CPU structure.
8903	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8904	* @thread EMT(pVCpu)
8905	*/
8906	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
8907	{
8908	AssertPtr(pVCpu);
8909	RT_NOREF1(fGlobalInit);
8910
8911	switch (enmEvent)
8912	{
8913	case RTTHREADCTXEVENT_OUT:
8914	{
8915	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8916	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8917	VMCPU_ASSERT_EMT(pVCpu);
8918
8919	/* No longjmps (logger flushes, locks) in this fragile context. */
8920	VMMRZCallRing3Disable(pVCpu);
8921	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8922
8923	/* Restore host-state (FPU, debug etc.) */
8924	if (!pVCpu->hm.s.fLeaveDone)
8925	{
8926	/*
8927	* Do -not- import the guest-state here as we might already be in the middle of importing
8928	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8929	*/
8930	hmR0VmxLeave(pVCpu, false /* fImportState */);
8931	pVCpu->hm.s.fLeaveDone = true;
8932	}
8933
8934	/* Leave HM context, takes care of local init (term). */
8935	int rc = HMR0LeaveCpu(pVCpu);
8936	AssertRC(rc);
8937
8938	/* Restore longjmp state. */
8939	VMMRZCallRing3Enable(pVCpu);
8940	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8941	break;
8942	}
8943
8944	case RTTHREADCTXEVENT_IN:
8945	{
8946	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8947	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8948	VMCPU_ASSERT_EMT(pVCpu);
8949
8950	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8951	VMMRZCallRing3Disable(pVCpu);
8952	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8953
8954	/* Initialize the bare minimum state required for HM. This takes care of
8955	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8956	int rc = hmR0EnterCpu(pVCpu);
8957	AssertRC(rc);
8958	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8959	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8960
8961	/* Load the active VMCS as the current one. */
8962	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8963	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8964	AssertRC(rc);
8965	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8966	pVCpu->hm.s.fLeaveDone = false;
8967
8968	/* Do the EMT scheduled L1D flush if needed. */
8969	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8970	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8971
8972	/* Restore longjmp state. */
8973	VMMRZCallRing3Enable(pVCpu);
8974	break;
8975	}
8976
8977	default:
8978	break;
8979	}
8980	}
8981
8982
8983	/**
8984	* Exports the host state into the VMCS host-state area.
8985	* Sets up the VM-exit MSR-load area.
8986	*
8987	* The CPU state will be loaded from these fields on every successful VM-exit.
8988	*
8989	* @returns VBox status code.
8990	* @param pVCpu The cross context virtual CPU structure.
8991	*
8992	* @remarks No-long-jump zone!!!
8993	*/
8994	static int hmR0VmxExportHostState(PVMCPUCC pVCpu)
8995	{
8996	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8997
8998	int rc = VINF_SUCCESS;
8999	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9000	{
9001	hmR0VmxExportHostControlRegs();
9002
9003	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9004	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9005
9006	hmR0VmxExportHostMsrs(pVCpu);
9007
9008	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9009	}
9010	return rc;
9011	}
9012
9013
9014	/**
9015	* Saves the host state in the VMCS host-state.
9016	*
9017	* @returns VBox status code.
9018	* @param pVCpu The cross context virtual CPU structure.
9019	*
9020	* @remarks No-long-jump zone!!!
9021	*/
9022	VMMR0DECL(int) VMXR0ExportHostState(PVMCPUCC pVCpu)
9023	{
9024	AssertPtr(pVCpu);
9025	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9026
9027	/*
9028	* Export the host state here while entering HM context.
9029	* When thread-context hooks are used, we might get preempted and have to re-save the host
9030	* state but most of the time we won't be, so do it here before we disable interrupts.
9031	*/
9032	return hmR0VmxExportHostState(pVCpu);
9033	}
9034
9035
9036	/**
9037	* Exports the guest state into the VMCS guest-state area.
9038	*
9039	* The will typically be done before VM-entry when the guest-CPU state and the
9040	* VMCS state may potentially be out of sync.
9041	*
9042	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9043	* VM-entry controls.
9044	* Sets up the appropriate VMX non-root function to execute guest code based on
9045	* the guest CPU mode.
9046	*
9047	* @returns VBox strict status code.
9048	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9049	* without unrestricted guest execution and the VMMDev is not presently
9050	* mapped (e.g. EFI32).
9051	*
9052	* @param pVCpu The cross context virtual CPU structure.
9053	* @param pVmxTransient The VMX-transient structure.
9054	*
9055	* @remarks No-long-jump zone!!!
9056	*/
9057	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9058	{
9059	AssertPtr(pVCpu);
9060	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9061	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9062
9063	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9064
9065	/*
9066	* Determine real-on-v86 mode.
9067	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9068	*/
9069	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9070	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9071	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9072	pVmcsInfo->RealMode. fRealOnV86Active = false;
9073	else
9074	{
9075	Assert(!pVmxTransient->fIsNestedGuest);
9076	pVmcsInfo->RealMode.fRealOnV86Active = true;
9077	}
9078
9079	/*
9080	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9081	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9082	*/
9083	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9084	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9085	* be a need to evaluate this everytime since I'm pretty sure we intercept
9086	* all guest paging mode changes. */
9087	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9088	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9089
9090	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9091	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9092
9093	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9094	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9095
9096	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9097	if (rcStrict == VINF_SUCCESS)
9098	{ /* likely */ }
9099	else
9100	{
9101	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9102	return rcStrict;
9103	}
9104
9105	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9106	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9107
9108	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9109	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9110
9111	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9112	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9113
9114	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9115	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9116
9117	rc = hmR0VmxExportGuestRip(pVCpu);
9118	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9119	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9120	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9121
9122	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9123	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9124
9125	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9126	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9127	\| HM_CHANGED_GUEST_CR2
9128	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9129	\| HM_CHANGED_GUEST_X87
9130	\| HM_CHANGED_GUEST_SSE_AVX
9131	\| HM_CHANGED_GUEST_OTHER_XSAVE
9132	\| HM_CHANGED_GUEST_XCRx
9133	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9134	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9135	\| HM_CHANGED_GUEST_TSC_AUX
9136	\| HM_CHANGED_GUEST_OTHER_MSRS
9137	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9138
9139	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9140	return rc;
9141	}
9142
9143
9144	/**
9145	* Exports the state shared between the host and guest into the VMCS.
9146	*
9147	* @param pVCpu The cross context virtual CPU structure.
9148	* @param pVmxTransient The VMX-transient structure.
9149	*
9150	* @remarks No-long-jump zone!!!
9151	*/
9152	static void hmR0VmxExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9153	{
9154	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9155	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9156
9157	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9158	{
9159	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9160	AssertRC(rc);
9161	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9162
9163	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9164	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9165	{
9166	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9167	AssertRC(rc);
9168	}
9169	}
9170
9171	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9172	{
9173	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9174	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9175	}
9176
9177	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9178	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9179	}
9180
9181
9182	/**
9183	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9184	*
9185	* @returns Strict VBox status code (i.e. informational status codes too).
9186	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9187	* without unrestricted guest execution and the VMMDev is not presently
9188	* mapped (e.g. EFI32).
9189	*
9190	* @param pVCpu The cross context virtual CPU structure.
9191	* @param pVmxTransient The VMX-transient structure.
9192	*
9193	* @remarks No-long-jump zone!!!
9194	*/
9195	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9196	{
9197	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9198	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9199	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9200
9201	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9202	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9203	#endif
9204
9205	/*
9206	* For many exits it's only RIP that changes and hence try to export it first
9207	* without going through a lot of change flag checks.
9208	*/
9209	VBOXSTRICTRC rcStrict;
9210	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9211	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9212	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9213	{
9214	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9215	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9216	{ /* likely */}
9217	else
9218	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9219	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9220	}
9221	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9222	{
9223	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9224	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9225	{ /* likely */}
9226	else
9227	{
9228	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9229	VBOXSTRICTRC_VAL(rcStrict)));
9230	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9231	return rcStrict;
9232	}
9233	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9234	}
9235	else
9236	rcStrict = VINF_SUCCESS;
9237
9238	#ifdef VBOX_STRICT
9239	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9240	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9241	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9242	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9243	("fCtxChanged=%#RX64\n", fCtxChanged));
9244	#endif
9245	return rcStrict;
9246	}
9247
9248
9249	/**
9250	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9251	* and update error record fields accordingly.
9252	*
9253	* @returns VMX_IGS_* error codes.
9254	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9255	* wrong with the guest state.
9256	*
9257	* @param pVCpu The cross context virtual CPU structure.
9258	* @param pVmcsInfo The VMCS info. object.
9259	*
9260	* @remarks This function assumes our cache of the VMCS controls
9261	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9262	*/
9263	static uint32_t hmR0VmxCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9264	{
9265	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9266	#define HMVMX_CHECK_BREAK(expr, err) do { \
9267	if (!(expr)) { uError = (err); break; } \
9268	} while (0)
9269
9270	int rc;
9271	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9272	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9273	uint32_t uError = VMX_IGS_ERROR;
9274	uint32_t u32Val;
9275	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9276
9277	do
9278	{
9279	/*
9280	* CR0.
9281	*/
9282	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9283	uint64_t fSetCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9284	uint64_t const fZapCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9285	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
9286	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9287	if (fUnrestrictedGuest)
9288	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
9289
9290	uint64_t u64GuestCr0;
9291	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64GuestCr0);
9292	AssertRC(rc);
9293	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9294	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9295	if ( !fUnrestrictedGuest
9296	&& (u64GuestCr0 & X86_CR0_PG)
9297	&& !(u64GuestCr0 & X86_CR0_PE))
9298	{
9299	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9300	}
9301
9302	/*
9303	* CR4.
9304	*/
9305	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9306	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9307	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9308
9309	uint64_t u64GuestCr4;
9310	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64GuestCr4);
9311	AssertRC(rc);
9312	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9313	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9314
9315	/*
9316	* IA32_DEBUGCTL MSR.
9317	*/
9318	uint64_t u64Val;
9319	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9320	AssertRC(rc);
9321	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9322	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9323	{
9324	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9325	}
9326	uint64_t u64DebugCtlMsr = u64Val;
9327
9328	#ifdef VBOX_STRICT
9329	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9330	AssertRC(rc);
9331	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9332	#endif
9333	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9334
9335	/*
9336	* RIP and RFLAGS.
9337	*/
9338	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
9339	AssertRC(rc);
9340	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9341	if ( !fLongModeGuest
9342	\|\| !pCtx->cs.Attr.n.u1Long)
9343	{
9344	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9345	}
9346	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9347	* must be identical if the "IA-32e mode guest" VM-entry
9348	* control is 1 and CS.L is 1. No check applies if the
9349	* CPU supports 64 linear-address bits. */
9350
9351	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9352	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9353	AssertRC(rc);
9354	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9355	VMX_IGS_RFLAGS_RESERVED);
9356	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9357	uint32_t const u32Eflags = u64Val;
9358
9359	if ( fLongModeGuest
9360	\|\| ( fUnrestrictedGuest
9361	&& !(u64GuestCr0 & X86_CR0_PE)))
9362	{
9363	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9364	}
9365
9366	uint32_t u32EntryInfo;
9367	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9368	AssertRC(rc);
9369	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9370	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9371
9372	/*
9373	* 64-bit checks.
9374	*/
9375	if (fLongModeGuest)
9376	{
9377	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9378	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9379	}
9380
9381	if ( !fLongModeGuest
9382	&& (u64GuestCr4 & X86_CR4_PCIDE))
9383	{
9384	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9385	}
9386
9387	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9388	* 51:32 beyond the processor's physical-address width are 0. */
9389
9390	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9391	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9392	{
9393	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9394	}
9395
9396	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9397	AssertRC(rc);
9398	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9399
9400	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9401	AssertRC(rc);
9402	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9403
9404	/*
9405	* PERF_GLOBAL MSR.
9406	*/
9407	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9408	{
9409	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9410	AssertRC(rc);
9411	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9412	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9413	}
9414
9415	/*
9416	* PAT MSR.
9417	*/
9418	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9419	{
9420	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9421	AssertRC(rc);
9422	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9423	for (unsigned i = 0; i < 8; i++)
9424	{
9425	uint8_t u8Val = (u64Val & 0xff);
9426	if ( u8Val != 0 /* UC */
9427	&& u8Val != 1 /* WC */
9428	&& u8Val != 4 /* WT */
9429	&& u8Val != 5 /* WP */
9430	&& u8Val != 6 /* WB */
9431	&& u8Val != 7 /* UC- */)
9432	{
9433	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9434	}
9435	u64Val >>= 8;
9436	}
9437	}
9438
9439	/*
9440	* EFER MSR.
9441	*/
9442	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9443	{
9444	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9445	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9446	AssertRC(rc);
9447	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9448	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9449	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9450	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9451	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9452	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9453	* iemVmxVmentryCheckGuestState(). */
9454	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9455	\|\| !(u64GuestCr0 & X86_CR0_PG)
9456	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9457	VMX_IGS_EFER_LMA_LME_MISMATCH);
9458	}
9459
9460	/*
9461	* Segment registers.
9462	*/
9463	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9464	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9465	if (!(u32Eflags & X86_EFL_VM))
9466	{
9467	/* CS */
9468	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9469	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9470	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9471	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9472	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9473	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9474	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9475	/* CS cannot be loaded with NULL in protected mode. */
9476	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9477	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9478	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9479	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9480	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9481	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9482	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9483	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9484	else
9485	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9486
9487	/* SS */
9488	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9489	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9490	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9491	if ( !(pCtx->cr0 & X86_CR0_PE)
9492	\|\| pCtx->cs.Attr.n.u4Type == 3)
9493	{
9494	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9495	}
9496	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9497	{
9498	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9499	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9500	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9501	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9502	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9503	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9504	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9505	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9506	}
9507
9508	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9509	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9510	{
9511	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9512	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9513	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9514	\|\| pCtx->ds.Attr.n.u4Type > 11
9515	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9516	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9517	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9518	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9519	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9520	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9521	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9522	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9523	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9524	}
9525	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9526	{
9527	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9528	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9529	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9530	\|\| pCtx->es.Attr.n.u4Type > 11
9531	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9532	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9533	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9534	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9535	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9536	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9537	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9538	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9539	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9540	}
9541	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9542	{
9543	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9544	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9545	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9546	\|\| pCtx->fs.Attr.n.u4Type > 11
9547	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9548	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9549	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9550	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9551	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9552	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9553	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9554	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9555	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9556	}
9557	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9558	{
9559	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9560	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9561	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9562	\|\| pCtx->gs.Attr.n.u4Type > 11
9563	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9564	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9565	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9566	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9567	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9568	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9569	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9570	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9571	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9572	}
9573	/* 64-bit capable CPUs. */
9574	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9575	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9576	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9577	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9578	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9579	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9580	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9581	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9582	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9583	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9584	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9585	}
9586	else
9587	{
9588	/* V86 mode checks. */
9589	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9590	if (pVmcsInfo->RealMode.fRealOnV86Active)
9591	{
9592	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9593	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9594	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9595	}
9596	else
9597	{
9598	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9599	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9600	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9601	}
9602
9603	/* CS */
9604	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9605	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9606	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9607	/* SS */
9608	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9609	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9610	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9611	/* DS */
9612	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9613	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9614	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9615	/* ES */
9616	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9617	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9618	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9619	/* FS */
9620	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9621	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9622	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9623	/* GS */
9624	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9625	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9626	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9627	/* 64-bit capable CPUs. */
9628	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9629	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9630	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9631	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9632	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9633	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9634	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9635	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9636	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9637	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9638	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9639	}
9640
9641	/*
9642	* TR.
9643	*/
9644	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9645	/* 64-bit capable CPUs. */
9646	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9647	if (fLongModeGuest)
9648	{
9649	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9650	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9651	}
9652	else
9653	{
9654	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9655	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9656	VMX_IGS_TR_ATTR_TYPE_INVALID);
9657	}
9658	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9659	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9660	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9661	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9662	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9663	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9664	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9665	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9666
9667	/*
9668	* GDTR and IDTR (64-bit capable checks).
9669	*/
9670	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9671	AssertRC(rc);
9672	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9673
9674	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9675	AssertRC(rc);
9676	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9677
9678	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9679	AssertRC(rc);
9680	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9681
9682	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9683	AssertRC(rc);
9684	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9685
9686	/*
9687	* Guest Non-Register State.
9688	*/
9689	/* Activity State. */
9690	uint32_t u32ActivityState;
9691	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9692	AssertRC(rc);
9693	HMVMX_CHECK_BREAK( !u32ActivityState
9694	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9695	VMX_IGS_ACTIVITY_STATE_INVALID);
9696	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9697	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9698	uint32_t u32IntrState;
9699	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9700	AssertRC(rc);
9701	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9702	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9703	{
9704	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9705	}
9706
9707	/** @todo Activity state and injecting interrupts. Left as a todo since we
9708	* currently don't use activity states but ACTIVE. */
9709
9710	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9711	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9712
9713	/* Guest interruptibility-state. */
9714	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9715	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9716	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9717	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9718	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9719	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9720	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9721	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9722	{
9723	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9724	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9725	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9726	}
9727	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9728	{
9729	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9730	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9731	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9732	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9733	}
9734	/** @todo Assumes the processor is not in SMM. */
9735	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9736	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9737	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9738	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9739	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9740	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9741	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9742	{
9743	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9744	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9745	}
9746
9747	/* Pending debug exceptions. */
9748	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9749	AssertRC(rc);
9750	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9751	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9752	u32Val = u64Val; /* For pending debug exceptions checks below. */
9753
9754	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9755	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9756	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9757	{
9758	if ( (u32Eflags & X86_EFL_TF)
9759	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9760	{
9761	/* Bit 14 is PendingDebug.BS. */
9762	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9763	}
9764	if ( !(u32Eflags & X86_EFL_TF)
9765	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9766	{
9767	/* Bit 14 is PendingDebug.BS. */
9768	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9769	}
9770	}
9771
9772	/* VMCS link pointer. */
9773	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9774	AssertRC(rc);
9775	if (u64Val != UINT64_C(0xffffffffffffffff))
9776	{
9777	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9778	/** @todo Bits beyond the processor's physical-address width MBZ. */
9779	/** @todo SMM checks. */
9780	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
9781	Assert(pVmcsInfo->pvShadowVmcs);
9782	VMXVMCSREVID VmcsRevId;
9783	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
9784	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
9785	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
9786	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
9787	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
9788	}
9789
9790	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9791	* not using nested paging? */
9792	if ( pVM->hm.s.fNestedPaging
9793	&& !fLongModeGuest
9794	&& CPUMIsGuestInPAEModeEx(pCtx))
9795	{
9796	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9797	AssertRC(rc);
9798	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9799
9800	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9801	AssertRC(rc);
9802	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9803
9804	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9805	AssertRC(rc);
9806	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9807
9808	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9809	AssertRC(rc);
9810	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9811	}
9812
9813	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9814	if (uError == VMX_IGS_ERROR)
9815	uError = VMX_IGS_REASON_NOT_FOUND;
9816	} while (0);
9817
9818	pVCpu->hm.s.u32HMError = uError;
9819	return uError;
9820
9821	#undef HMVMX_ERROR_BREAK
9822	#undef HMVMX_CHECK_BREAK
9823	}
9824
9825
9826	/**
9827	* Map the APIC-access page for virtualizing APIC accesses.
9828	*
9829	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
9830	* this not done as part of exporting guest state, see @bugref{8721}.
9831	*
9832	* @returns VBox status code.
9833	* @param pVCpu The cross context virtual CPU structure.
9834	*/
9835	static int hmR0VmxMapHCApicAccessPage(PVMCPUCC pVCpu)
9836	{
9837	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9838	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9839
9840	Assert(PDMHasApic(pVM));
9841	Assert(u64MsrApicBase);
9842
9843	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9844	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9845
9846	/* Unalias the existing mapping. */
9847	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9848	AssertRCReturn(rc, rc);
9849
9850	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9851	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9852	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9853	AssertRCReturn(rc, rc);
9854
9855	/* Update the per-VCPU cache of the APIC base MSR. */
9856	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9857	return VINF_SUCCESS;
9858	}
9859
9860
9861	/**
9862	* Worker function passed to RTMpOnSpecific() that is to be called on the target
9863	* CPU.
9864	*
9865	* @param idCpu The ID for the CPU the function is called on.
9866	* @param pvUser1 Null, not used.
9867	* @param pvUser2 Null, not used.
9868	*/
9869	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
9870	{
9871	RT_NOREF3(idCpu, pvUser1, pvUser2);
9872	VMXDispatchHostNmi();
9873	}
9874
9875
9876	/**
9877	* Dispatching an NMI on the host CPU that received it.
9878	*
9879	* @returns VBox status code.
9880	* @param pVCpu The cross context virtual CPU structure.
9881	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
9882	* executing when receiving the host NMI in VMX non-root
9883	* operation.
9884	*/
9885	static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9886	{
9887	RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
9888	Assert(idCpu != NIL_RTCPUID);
9889
9890	/*
9891	* We don't want to delay dispatching the NMI any more than we have to. However,
9892	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
9893	* after executing guest or nested-guest code for the following reasons:
9894	*
9895	* - We would need to perform VMREADs with interrupts disabled and is orders of
9896	* magnitude worse when we run as a nested hypervisor without VMCS shadowing
9897	* supported by the host hypervisor.
9898	*
9899	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
9900	* longer period of time just for handling an edge case like host NMIs which do
9901	* not occur nearly as frequently as other VM-exits.
9902	*
9903	* Let's cover the most likely scenario first. Check if we are on the target CPU
9904	* and dispatch the NMI right away. This should be much faster than calling into
9905	* RTMpOnSpecific() machinery.
9906	*/
9907	bool fDispatched = false;
9908	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
9909	if (idCpu == RTMpCpuId())
9910	{
9911	VMXDispatchHostNmi();
9912	fDispatched = true;
9913	}
9914	ASMSetFlags(fEFlags);
9915	if (fDispatched)
9916	{
9917	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9918	return VINF_SUCCESS;
9919	}
9920
9921	/*
9922	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
9923	* there should be no race or recursion even if we are unlucky enough to be preempted
9924	* (to the target CPU) without dispatching the host NMI above.
9925	*/
9926	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
9927	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
9928	}
9929
9930
9931	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9932	/**
9933	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9934	* nested-guest using hardware-assisted VMX.
9935	*
9936	* @param pVCpu The cross context virtual CPU structure.
9937	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9938	* @param pVmcsInfoGst The guest VMCS info. object.
9939	*/
9940	static void hmR0VmxMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9941	{
9942	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
9943	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9944	Assert(pu64MsrBitmap);
9945
9946	/*
9947	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9948	* MSR that is intercepted by the guest is also intercepted while executing the
9949	* nested-guest using hardware-assisted VMX.
9950	*
9951	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
9952	* nested-guest VM-exit even if the outer guest is not intercepting some
9953	* MSRs. We cannot assume the caller has initialized the nested-guest
9954	* MSR bitmap in this case.
9955	*
9956	* The nested hypervisor may also switch whether it uses MSR bitmaps for
9957	* each VM-entry, hence initializing it once per-VM while setting up the
9958	* nested-guest VMCS is not sufficient.
9959	*/
9960	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9961	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
9962	{
9963	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9964	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9965	Assert(pu64MsrBitmapNstGst);
9966	Assert(pu64MsrBitmapGst);
9967
9968	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
9969	for (uint32_t i = 0; i < cFrags; i++)
9970	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9971	}
9972	else
9973	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
9974	}
9975
9976
9977	/**
9978	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9979	* hardware-assisted VMX execution of the nested-guest.
9980	*
9981	* For a guest, we don't modify these controls once we set up the VMCS and hence
9982	* this function is never called.
9983	*
9984	* For nested-guests since the nested hypervisor provides these controls on every
9985	* nested-guest VM-entry and could potentially change them everytime we need to
9986	* merge them before every nested-guest VM-entry.
9987	*
9988	* @returns VBox status code.
9989	* @param pVCpu The cross context virtual CPU structure.
9990	*/
9991	static int hmR0VmxMergeVmcsNested(PVMCPUCC pVCpu)
9992	{
9993	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9994	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9995	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9996	Assert(pVmcsNstGst);
9997
9998	/*
9999	* Merge the controls with the requirements of the guest VMCS.
10000	*
10001	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10002	* VMCS with the features supported by the physical CPU as it's already done by the
10003	* VMLAUNCH/VMRESUME instruction emulation.
10004	*
10005	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10006	* derived from the VMX features supported by the physical CPU.
10007	*/
10008
10009	/* Pin-based VM-execution controls. */
10010	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10011
10012	/* Processor-based VM-execution controls. */
10013	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10014	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10015	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10016	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10017	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10018
10019	/* Secondary processor-based VM-execution controls. */
10020	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10021	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10022	\| VMX_PROC_CTLS2_INVPCID
10023	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10024	\| VMX_PROC_CTLS2_RDTSCP
10025	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10026	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10027	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10028	\| VMX_PROC_CTLS2_VMFUNC));
10029
10030	/*
10031	* VM-entry controls:
10032	* These controls contains state that depends on the nested-guest state (primarily
10033	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10034	* VM-exit. Although the nested hypervisor cannot change it, we need to in order to
10035	* properly continue executing the nested-guest if the EFER MSR changes but does not
10036	* cause a nested-guest VM-exits.
10037	*
10038	* VM-exit controls:
10039	* These controls specify the host state on return. We cannot use the controls from
10040	* the nested hypervisor state as is as it would contain the guest state rather than
10041	* the host state. Since the host state is subject to change (e.g. preemption, trips
10042	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10043	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10044	*
10045	* VM-entry MSR-load:
10046	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10047	* context by the VMLAUNCH/VMRESUME instruction emulation.
10048	*
10049	* VM-exit MSR-store:
10050	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10051	* back into the VM-exit MSR-store area.
10052	*
10053	* VM-exit MSR-load areas:
10054	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10055	* can entirely ignore what the nested hypervisor wants to load here.
10056	*/
10057
10058	/*
10059	* Exception bitmap.
10060	*
10061	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10062	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10063	* code more flexible if intercepting exceptions become more dynamic in the future we do
10064	* it as part of exporting the nested-guest state.
10065	*/
10066	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10067
10068	/*
10069	* CR0/CR4 guest/host mask.
10070	*
10071	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10072	* cause VM-exits, so we need to merge them here.
10073	*/
10074	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10075	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10076
10077	/*
10078	* Page-fault error-code mask and match.
10079	*
10080	* Although we require unrestricted guest execution (and thereby nested-paging) for
10081	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10082	* normally intercept #PFs, it might intercept them for debugging purposes.
10083	*
10084	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10085	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10086	*/
10087	uint32_t u32XcptPFMask;
10088	uint32_t u32XcptPFMatch;
10089	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10090	{
10091	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10092	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10093	}
10094	else
10095	{
10096	u32XcptPFMask = 0;
10097	u32XcptPFMatch = 0;
10098	}
10099
10100	/*
10101	* Pause-Loop exiting.
10102	*/
10103	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10104	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10105
10106	/*
10107	* Pending debug exceptions.
10108	* Currently just copy whatever the nested-guest provides us.
10109	*/
10110	uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
10111
10112	/*
10113	* I/O Bitmap.
10114	*
10115	* We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
10116	* intercept all I/O port accesses.
10117	*/
10118	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10119	Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
10120
10121	/*
10122	* VMCS shadowing.
10123	*
10124	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10125	* enabled while executing the nested-guest.
10126	*/
10127	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10128
10129	/*
10130	* APIC-access page.
10131	*/
10132	RTHCPHYS HCPhysApicAccess;
10133	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10134	{
10135	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10136	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10137
10138	/** @todo NSTVMX: This is not really correct but currently is required to make
10139	* things work. We need to re-enable the page handler when we fallback to
10140	* IEM execution of the nested-guest! */
10141	PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
10142
10143	void *pvPage;
10144	PGMPAGEMAPLOCK PgLockApicAccess;
10145	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
10146	if (RT_SUCCESS(rc))
10147	{
10148	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10149	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10150
10151	/** @todo Handle proper releasing of page-mapping lock later. */
10152	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
10153	}
10154	else
10155	return rc;
10156	}
10157	else
10158	HCPhysApicAccess = 0;
10159
10160	/*
10161	* Virtual-APIC page and TPR threshold.
10162	*/
10163	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10164	RTHCPHYS HCPhysVirtApic;
10165	uint32_t u32TprThreshold;
10166	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10167	{
10168	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10169	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10170
10171	void *pvPage;
10172	PGMPAGEMAPLOCK PgLockVirtApic;
10173	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
10174	if (RT_SUCCESS(rc))
10175	{
10176	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10177	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10178
10179	/** @todo Handle proper releasing of page-mapping lock later. */
10180	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
10181	}
10182	else
10183	return rc;
10184
10185	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10186	}
10187	else
10188	{
10189	HCPhysVirtApic = 0;
10190	u32TprThreshold = 0;
10191
10192	/*
10193	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10194	* used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
10195	* be taken care of by EPT/shadow paging.
10196	*/
10197	if (pVM->hm.s.fAllow64BitGuests)
10198	{
10199	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10200	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10201	}
10202	}
10203
10204	/*
10205	* Validate basic assumptions.
10206	*/
10207	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10208	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10209	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10210
10211	/*
10212	* Commit it to the nested-guest VMCS.
10213	*/
10214	int rc = VINF_SUCCESS;
10215	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10216	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10217	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10218	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10219	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10220	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10221	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10222	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10223	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10224	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10225	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10226	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10227	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10228	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10229	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10230	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10231	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10232	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10233	{
10234	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10235	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10236	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10237	}
10238	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10239	{
10240	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10241	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10242	}
10243	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10244	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10245	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
10246	AssertRC(rc);
10247
10248	/*
10249	* Update the nested-guest VMCS cache.
10250	*/
10251	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10252	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10253	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10254	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10255	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10256	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10257	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10258	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10259	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10260
10261	/*
10262	* We need to flush the TLB if we are switching the APIC-access page address.
10263	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10264	*/
10265	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10266	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10267
10268	/*
10269	* MSR bitmap.
10270	*
10271	* The MSR bitmap address has already been initialized while setting up the nested-guest
10272	* VMCS, here we need to merge the MSR bitmaps.
10273	*/
10274	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10275	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10276
10277	return VINF_SUCCESS;
10278	}
10279	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10280
10281
10282	/**
10283	* Does the preparations before executing guest code in VT-x.
10284	*
10285	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10286	* recompiler/IEM. We must be cautious what we do here regarding committing
10287	* guest-state information into the VMCS assuming we assuredly execute the
10288	* guest in VT-x mode.
10289	*
10290	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10291	* the common-state (TRPM/forceflags), we must undo those changes so that the
10292	* recompiler/IEM can (and should) use them when it resumes guest execution.
10293	* Otherwise such operations must be done when we can no longer exit to ring-3.
10294	*
10295	* @returns Strict VBox status code (i.e. informational status codes too).
10296	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10297	* have been disabled.
10298	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10299	* pending events).
10300	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10301	* double-fault into the guest.
10302	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10303	* dispatched directly.
10304	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10305	*
10306	* @param pVCpu The cross context virtual CPU structure.
10307	* @param pVmxTransient The VMX-transient structure.
10308	* @param fStepping Whether we are single-stepping the guest in the
10309	* hypervisor debugger. Makes us ignore some of the reasons
10310	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10311	* if event dispatching took place.
10312	*/
10313	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10314	{
10315	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10316
10317	Log4Func(("fIsNested=%RTbool fStepping=%RTbool\n", pVmxTransient->fIsNestedGuest, fStepping));
10318
10319	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10320	if (pVmxTransient->fIsNestedGuest)
10321	{
10322	RT_NOREF2(pVCpu, fStepping);
10323	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10324	return VINF_EM_RESCHEDULE_REM;
10325	}
10326	#endif
10327
10328	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10329	PGMRZDynMapFlushAutoSet(pVCpu);
10330	#endif
10331
10332	/*
10333	* Check and process force flag actions, some of which might require us to go back to ring-3.
10334	*/
10335	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10336	if (rcStrict == VINF_SUCCESS)
10337	{
10338	/* FFs don't get set all the time. */
10339	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10340	if ( pVmxTransient->fIsNestedGuest
10341	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10342	{
10343	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10344	return VINF_VMX_VMEXIT;
10345	}
10346	#endif
10347	}
10348	else
10349	return rcStrict;
10350
10351	/*
10352	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10353	*/
10354	/** @todo Doing this from ring-3 after VM setup phase causes a
10355	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
10356	* idea why atm. */
10357	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10358	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10359	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10360	&& PDMHasApic(pVM))
10361	{
10362	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10363	AssertRCReturn(rc, rc);
10364	}
10365
10366	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10367	/*
10368	* Merge guest VMCS controls with the nested-guest VMCS controls.
10369	*
10370	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10371	* saved state), we should be okay with merging controls as we initialize the
10372	* guest VMCS controls as part of VM setup phase.
10373	*/
10374	if ( pVmxTransient->fIsNestedGuest
10375	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10376	{
10377	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10378	AssertRCReturn(rc, rc);
10379	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10380	}
10381	#endif
10382
10383	/*
10384	* Evaluate events to be injected into the guest.
10385	*
10386	* Events in TRPM can be injected without inspecting the guest state.
10387	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10388	* guest to cause a VM-exit the next time they are ready to receive the event.
10389	*
10390	* With nested-guests, evaluating pending events may cause VM-exits. Also, verify
10391	* that the event in TRPM that we will inject using hardware-assisted VMX is -not-
10392	* subject to interecption. Otherwise, we should have checked and injected them
10393	* manually elsewhere (IEM).
10394	*/
10395	if (TRPMHasTrap(pVCpu))
10396	{
10397	Assert(!pVmxTransient->fIsNestedGuest \|\| !CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
10398	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10399	}
10400
10401	uint32_t fIntrState;
10402	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10403
10404	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10405	/*
10406	* While evaluating pending events if something failed (unlikely) or if we were
10407	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10408	*/
10409	if (rcStrict != VINF_SUCCESS)
10410	return rcStrict;
10411	if ( pVmxTransient->fIsNestedGuest
10412	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10413	{
10414	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10415	return VINF_VMX_VMEXIT;
10416	}
10417	#else
10418	Assert(rcStrict == VINF_SUCCESS);
10419	#endif
10420
10421	/*
10422	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10423	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10424	* also result in triple-faulting the VM.
10425	*
10426	* With nested-guests, the above does not apply since unrestricted guest execution is a
10427	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10428	*/
10429	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10430	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10431	{ /* likely */ }
10432	else
10433	{
10434	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10435	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10436	return rcStrict;
10437	}
10438
10439	/*
10440	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10441	* import CR3 themselves. We will need to update them here, as even as late as the above
10442	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10443	* the below force flags to be set.
10444	*/
10445	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10446	{
10447	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10448	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10449	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10450	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10451	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10452	}
10453	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10454	{
10455	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10456	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10457	}
10458
10459	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10460	/* Paranoia. */
10461	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10462	#endif
10463
10464	/*
10465	* No longjmps to ring-3 from this point on!!!
10466	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10467	* This also disables flushing of the R0-logger instance (if any).
10468	*/
10469	VMMRZCallRing3Disable(pVCpu);
10470
10471	/*
10472	* Export the guest state bits.
10473	*
10474	* We cannot perform longjmps while loading the guest state because we do not preserve the
10475	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10476	* CPU migration.
10477	*
10478	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10479	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10480	*/
10481	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10482	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10483	{ /* likely */ }
10484	else
10485	{
10486	VMMRZCallRing3Enable(pVCpu);
10487	return rcStrict;
10488	}
10489
10490	/*
10491	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10492	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10493	* preemption disabled for a while. Since this is purely to aid the
10494	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10495	* disable interrupt on NT.
10496	*
10497	* We need to check for force-flags that could've possible been altered since we last
10498	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10499	* see @bugref{6398}).
10500	*
10501	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10502	* to ring-3 before executing guest code.
10503	*/
10504	pVmxTransient->fEFlags = ASMIntDisableFlags();
10505
10506	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10507	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10508	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10509	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10510	{
10511	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10512	{
10513	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10514	/*
10515	* If we are executing a nested-guest make sure that we should intercept subsequent
10516	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10517	* the VM-exit instruction emulation happy.
10518	*/
10519	if (pVmxTransient->fIsNestedGuest)
10520	CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
10521	#endif
10522
10523	/*
10524	* We've injected any pending events. This is really the point of no return (to ring-3).
10525	*
10526	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10527	* returns from this function, so do -not- enable them here.
10528	*/
10529	pVCpu->hm.s.Event.fPending = false;
10530	return VINF_SUCCESS;
10531	}
10532
10533	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10534	rcStrict = VINF_EM_RAW_INTERRUPT;
10535	}
10536	else
10537	{
10538	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10539	rcStrict = VINF_EM_RAW_TO_R3;
10540	}
10541
10542	ASMSetFlags(pVmxTransient->fEFlags);
10543	VMMRZCallRing3Enable(pVCpu);
10544
10545	return rcStrict;
10546	}
10547
10548
10549	/**
10550	* Final preparations before executing guest code using hardware-assisted VMX.
10551	*
10552	* We can no longer get preempted to a different host CPU and there are no returns
10553	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10554	* failures), this function is not intended to fail sans unrecoverable hardware
10555	* errors.
10556	*
10557	* @param pVCpu The cross context virtual CPU structure.
10558	* @param pVmxTransient The VMX-transient structure.
10559	*
10560	* @remarks Called with preemption disabled.
10561	* @remarks No-long-jump zone!!!
10562	*/
10563	static void hmR0VmxPreRunGuestCommitted(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10564	{
10565	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10566	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10567	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10568	Assert(!pVCpu->hm.s.Event.fPending);
10569
10570	/*
10571	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10572	*/
10573	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10574	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10575
10576	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10577	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10578	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10579	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10580
10581	if (!CPUMIsGuestFPUStateActive(pVCpu))
10582	{
10583	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10584	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10585	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10586	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10587	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10588	}
10589
10590	/*
10591	* Re-export the host state bits as we may've been preempted (only happens when
10592	* thread-context hooks are used or when the VM start function changes) or if
10593	* the host CR0 is modified while loading the guest FPU state above.
10594	*
10595	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10596	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10597	* see @bugref{8432}.
10598	*
10599	* This may also happen when switching to/from a nested-guest VMCS without leaving
10600	* ring-0.
10601	*/
10602	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10603	{
10604	hmR0VmxExportHostState(pVCpu);
10605	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10606	}
10607	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10608
10609	/*
10610	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10611	*/
10612	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10613	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10614	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10615
10616	/*
10617	* Store status of the shared guest/host debug state at the time of VM-entry.
10618	*/
10619	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10620	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10621
10622	/*
10623	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10624	* more than one conditional check. The post-run side of our code shall determine
10625	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10626	*/
10627	if (pVmcsInfo->pbVirtApic)
10628	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10629
10630	/*
10631	* Update the host MSRs values in the VM-exit MSR-load area.
10632	*/
10633	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10634	{
10635	if (pVmcsInfo->cExitMsrLoad > 0)
10636	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10637	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10638	}
10639
10640	/*
10641	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10642	* VMX-preemption timer based on the next virtual sync clock deadline.
10643	*/
10644	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10645	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10646	{
10647	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10648	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10649	}
10650
10651	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10652	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10653	if (!fIsRdtscIntercepted)
10654	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10655	else
10656	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10657
10658	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10659	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10660	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10661	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Record the error reporting info. with the current host CPU. */
10662	pVmcsInfo->idHostCpuState = idCurrentCpu; /* Record the CPU for which the host-state has been exported. */
10663	pVmcsInfo->idHostCpuExec = idCurrentCpu; /* Record the CPU on which we shall execute. */
10664
10665	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10666
10667	TMNotifyStartOfExecution(pVM, pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10668	as we're about to start executing the guest. */
10669
10670	/*
10671	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10672	*
10673	* This is done this late as updating the TSC offsetting/preemption timer above
10674	* figures out if we can skip intercepting RDTSCP by calculating the number of
10675	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10676	*/
10677	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10678	&& !fIsRdtscIntercepted)
10679	{
10680	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10681
10682	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10683	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10684	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10685	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10686	AssertRC(rc);
10687	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10688	pVmxTransient->fRemoveTscAuxMsr = true;
10689	}
10690
10691	#ifdef VBOX_STRICT
10692	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10693	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10694	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10695	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10696	#endif
10697
10698	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10699	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10700	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10701	* see @bugref{9180#c54}. */
10702	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10703	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10704	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10705	#endif
10706	}
10707
10708
10709	/**
10710	* First C routine invoked after running guest code using hardware-assisted VMX.
10711	*
10712	* @param pVCpu The cross context virtual CPU structure.
10713	* @param pVmxTransient The VMX-transient structure.
10714	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10715	*
10716	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10717	*
10718	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10719	* unconditionally when it is safe to do so.
10720	*/
10721	static void hmR0VmxPostRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10722	{
10723	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10724
10725	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10726	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10727	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10728	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10729	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10730	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10731
10732	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10733	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10734	{
10735	uint64_t uGstTsc;
10736	if (!pVmxTransient->fIsNestedGuest)
10737	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10738	else
10739	{
10740	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10741	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10742	}
10743	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10744	}
10745
10746	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10747	TMNotifyEndOfExecution(pVCpu->CTX_SUFF(pVM), pVCpu); /* Notify TM that the guest is no longer running. */
10748	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10749
10750	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10751	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10752	#ifdef VBOX_STRICT
10753	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10754	#endif
10755	Assert(!ASMIntAreEnabled());
10756	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10757	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10758
10759	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
10760	/*
10761	* Clean all the VMCS fields in the transient structure before reading
10762	* anything from the VMCS.
10763	*/
10764	pVmxTransient->uExitReason = 0;
10765	pVmxTransient->uExitIntErrorCode = 0;
10766	pVmxTransient->uExitQual = 0;
10767	pVmxTransient->uGuestLinearAddr = 0;
10768	pVmxTransient->uExitIntInfo = 0;
10769	pVmxTransient->cbExitInstr = 0;
10770	pVmxTransient->ExitInstrInfo.u = 0;
10771	pVmxTransient->uEntryIntInfo = 0;
10772	pVmxTransient->uEntryXcptErrorCode = 0;
10773	pVmxTransient->cbEntryInstr = 0;
10774	pVmxTransient->uIdtVectoringInfo = 0;
10775	pVmxTransient->uIdtVectoringErrorCode = 0;
10776	#endif
10777
10778	/*
10779	* Save the basic VM-exit reason and check if the VM-entry failed.
10780	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10781	*/
10782	uint32_t uExitReason;
10783	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10784	AssertRC(rc);
10785	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10786	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10787
10788	/*
10789	* Log the VM-exit before logging anything else as otherwise it might be a
10790	* tad confusing what happens before and after the world-switch.
10791	*/
10792	HMVMX_LOG_EXIT(pVCpu, uExitReason);
10793
10794	/*
10795	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
10796	* bitmap permissions, if it was added before VM-entry.
10797	*/
10798	if (pVmxTransient->fRemoveTscAuxMsr)
10799	{
10800	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10801	pVmxTransient->fRemoveTscAuxMsr = false;
10802	}
10803
10804	/*
10805	* Check if VMLAUNCH/VMRESUME succeeded.
10806	* If this failed, we cause a guru meditation and cease further execution.
10807	*
10808	* However, if we are executing a nested-guest we might fail if we use the
10809	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10810	*/
10811	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10812	{
10813	/*
10814	* Update the VM-exit history array here even if the VM-entry failed due to:
10815	* - Invalid guest state.
10816	* - MSR loading.
10817	* - Machine-check event.
10818	*
10819	* In any of the above cases we will still have a "valid" VM-exit reason
10820	* despite @a fVMEntryFailed being false.
10821	*
10822	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10823	*
10824	* Note! We don't have CS or RIP at this point. Will probably address that later
10825	* by amending the history entry added here.
10826	*/
10827	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10828	UINT64_MAX, uHostTsc);
10829
10830	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10831	{
10832	VMMRZCallRing3Enable(pVCpu);
10833
10834	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10835	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10836
10837	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
10838	hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
10839	#endif
10840	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10841	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10842	AssertRC(rc);
10843	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10844	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10845	AssertRC(rc);
10846	#else
10847	/*
10848	* Import the guest-interruptibility state always as we need it while evaluating
10849	* injecting events on re-entry.
10850	*
10851	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10852	* checking for real-mode while exporting the state because all bits that cause
10853	* mode changes wrt CR0 are intercepted.
10854	*/
10855	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10856	AssertRC(rc);
10857	#endif
10858
10859	/*
10860	* Sync the TPR shadow with our APIC state.
10861	*/
10862	if ( !pVmxTransient->fIsNestedGuest
10863	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10864	{
10865	Assert(pVmcsInfo->pbVirtApic);
10866	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10867	{
10868	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10869	AssertRC(rc);
10870	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10871	}
10872	}
10873
10874	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10875	return;
10876	}
10877	}
10878	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10879	else if (pVmxTransient->fIsNestedGuest)
10880	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10881	#endif
10882	else
10883	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10884
10885	VMMRZCallRing3Enable(pVCpu);
10886	}
10887
10888
10889	/**
10890	* Runs the guest code using hardware-assisted VMX the normal way.
10891	*
10892	* @returns VBox status code.
10893	* @param pVCpu The cross context virtual CPU structure.
10894	* @param pcLoops Pointer to the number of executed loops.
10895	*/
10896	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
10897	{
10898	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10899	Assert(pcLoops);
10900	Assert(*pcLoops <= cMaxResumeLoops);
10901	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10902
10903	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10904	/*
10905	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
10906	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
10907	* guest VMCS while entering the VMX ring-0 session.
10908	*/
10909	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10910	{
10911	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
10912	if (RT_SUCCESS(rc))
10913	{ /* likely */ }
10914	else
10915	{
10916	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
10917	return rc;
10918	}
10919	}
10920	#endif
10921
10922	VMXTRANSIENT VmxTransient;
10923	RT_ZERO(VmxTransient);
10924	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10925
10926	/* Paranoia. */
10927	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10928
10929	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10930	for (;;)
10931	{
10932	Assert(!HMR0SuspendPending());
10933	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10934	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10935
10936	/*
10937	* Preparatory work for running nested-guest code, this may force us to
10938	* return to ring-3.
10939	*
10940	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10941	*/
10942	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10943	if (rcStrict != VINF_SUCCESS)
10944	break;
10945
10946	/* Interrupts are disabled at this point! */
10947	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10948	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10949	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10950	/* Interrupts are re-enabled at this point! */
10951
10952	/*
10953	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10954	*/
10955	if (RT_SUCCESS(rcRun))
10956	{ /* very likely */ }
10957	else
10958	{
10959	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10960	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10961	return rcRun;
10962	}
10963
10964	/*
10965	* Profile the VM-exit.
10966	*/
10967	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10968	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10969	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10970	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10971	HMVMX_START_EXIT_DISPATCH_PROF();
10972
10973	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10974
10975	/*
10976	* Handle the VM-exit.
10977	*/
10978	#ifdef HMVMX_USE_FUNCTION_TABLE
10979	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10980	#else
10981	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10982	#endif
10983	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10984	if (rcStrict == VINF_SUCCESS)
10985	{
10986	if (++(*pcLoops) <= cMaxResumeLoops)
10987	continue;
10988	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10989	rcStrict = VINF_EM_RAW_INTERRUPT;
10990	}
10991	break;
10992	}
10993
10994	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10995	return rcStrict;
10996	}
10997
10998
10999	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11000	/**
11001	* Runs the nested-guest code using hardware-assisted VMX.
11002	*
11003	* @returns VBox status code.
11004	* @param pVCpu The cross context virtual CPU structure.
11005	* @param pcLoops Pointer to the number of executed loops.
11006	*
11007	* @sa hmR0VmxRunGuestCodeNormal.
11008	*/
11009	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
11010	{
11011	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11012	Assert(pcLoops);
11013	Assert(*pcLoops <= cMaxResumeLoops);
11014	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11015
11016	/*
11017	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11018	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11019	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11020	*/
11021	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11022	{
11023	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11024	if (RT_SUCCESS(rc))
11025	{ /* likely */ }
11026	else
11027	{
11028	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11029	return rc;
11030	}
11031	}
11032
11033	VMXTRANSIENT VmxTransient;
11034	RT_ZERO(VmxTransient);
11035	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11036	VmxTransient.fIsNestedGuest = true;
11037
11038	/* Paranoia. */
11039	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11040
11041	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11042	for (;;)
11043	{
11044	Assert(!HMR0SuspendPending());
11045	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11046	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11047
11048	/*
11049	* Preparatory work for running guest code, this may force us to
11050	* return to ring-3.
11051	*
11052	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11053	*/
11054	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11055	if (rcStrict != VINF_SUCCESS)
11056	break;
11057
11058	/* Interrupts are disabled at this point! */
11059	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11060	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11061	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11062	/* Interrupts are re-enabled at this point! */
11063
11064	/*
11065	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11066	*/
11067	if (RT_SUCCESS(rcRun))
11068	{ /* very likely */ }
11069	else
11070	{
11071	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11072	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11073	return rcRun;
11074	}
11075
11076	/*
11077	* Profile the VM-exit.
11078	*/
11079	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11080	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11081	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11082	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11083	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11084	HMVMX_START_EXIT_DISPATCH_PROF();
11085
11086	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11087
11088	/*
11089	* Handle the VM-exit.
11090	*/
11091	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11092	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11093	if (rcStrict == VINF_SUCCESS)
11094	{
11095	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11096	{
11097	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11098	rcStrict = VINF_VMX_VMEXIT;
11099	}
11100	else
11101	{
11102	if (++(*pcLoops) <= cMaxResumeLoops)
11103	continue;
11104	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11105	rcStrict = VINF_EM_RAW_INTERRUPT;
11106	}
11107	}
11108	else
11109	Assert(rcStrict != VINF_VMX_VMEXIT);
11110	break;
11111	}
11112
11113	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11114	return rcStrict;
11115	}
11116	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11117
11118
11119	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11120	* probes.
11121	*
11122	* The following few functions and associated structure contains the bloat
11123	* necessary for providing detailed debug events and dtrace probes as well as
11124	* reliable host side single stepping. This works on the principle of
11125	* "subclassing" the normal execution loop and workers. We replace the loop
11126	* method completely and override selected helpers to add necessary adjustments
11127	* to their core operation.
11128	*
11129	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11130	* any performance for debug and analysis features.
11131	*
11132	* @{
11133	*/
11134
11135	/**
11136	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11137	* the debug run loop.
11138	*/
11139	typedef struct VMXRUNDBGSTATE
11140	{
11141	/** The RIP we started executing at. This is for detecting that we stepped. */
11142	uint64_t uRipStart;
11143	/** The CS we started executing with. */
11144	uint16_t uCsStart;
11145
11146	/** Whether we've actually modified the 1st execution control field. */
11147	bool fModifiedProcCtls : 1;
11148	/** Whether we've actually modified the 2nd execution control field. */
11149	bool fModifiedProcCtls2 : 1;
11150	/** Whether we've actually modified the exception bitmap. */
11151	bool fModifiedXcptBitmap : 1;
11152
11153	/** We desire the modified the CR0 mask to be cleared. */
11154	bool fClearCr0Mask : 1;
11155	/** We desire the modified the CR4 mask to be cleared. */
11156	bool fClearCr4Mask : 1;
11157	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11158	uint32_t fCpe1Extra;
11159	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11160	uint32_t fCpe1Unwanted;
11161	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11162	uint32_t fCpe2Extra;
11163	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11164	uint32_t bmXcptExtra;
11165	/** The sequence number of the Dtrace provider settings the state was
11166	* configured against. */
11167	uint32_t uDtraceSettingsSeqNo;
11168	/** VM-exits to check (one bit per VM-exit). */
11169	uint32_t bmExitsToCheck[3];
11170
11171	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11172	uint32_t fProcCtlsInitial;
11173	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11174	uint32_t fProcCtls2Initial;
11175	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11176	uint32_t bmXcptInitial;
11177	} VMXRUNDBGSTATE;
11178	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11179	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11180
11181
11182	/**
11183	* Initializes the VMXRUNDBGSTATE structure.
11184	*
11185	* @param pVCpu The cross context virtual CPU structure of the
11186	* calling EMT.
11187	* @param pVmxTransient The VMX-transient structure.
11188	* @param pDbgState The debug state to initialize.
11189	*/
11190	static void hmR0VmxRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11191	{
11192	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11193	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11194
11195	pDbgState->fModifiedProcCtls = false;
11196	pDbgState->fModifiedProcCtls2 = false;
11197	pDbgState->fModifiedXcptBitmap = false;
11198	pDbgState->fClearCr0Mask = false;
11199	pDbgState->fClearCr4Mask = false;
11200	pDbgState->fCpe1Extra = 0;
11201	pDbgState->fCpe1Unwanted = 0;
11202	pDbgState->fCpe2Extra = 0;
11203	pDbgState->bmXcptExtra = 0;
11204	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11205	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11206	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11207	}
11208
11209
11210	/**
11211	* Updates the VMSC fields with changes requested by @a pDbgState.
11212	*
11213	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11214	* immediately before executing guest code, i.e. when interrupts are disabled.
11215	* We don't check status codes here as we cannot easily assert or return in the
11216	* latter case.
11217	*
11218	* @param pVCpu The cross context virtual CPU structure.
11219	* @param pVmxTransient The VMX-transient structure.
11220	* @param pDbgState The debug state.
11221	*/
11222	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11223	{
11224	/*
11225	* Ensure desired flags in VMCS control fields are set.
11226	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11227	*
11228	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11229	* there should be no stale data in pCtx at this point.
11230	*/
11231	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11232	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11233	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11234	{
11235	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11236	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11237	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11238	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11239	pDbgState->fModifiedProcCtls = true;
11240	}
11241
11242	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11243	{
11244	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11245	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11246	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11247	pDbgState->fModifiedProcCtls2 = true;
11248	}
11249
11250	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11251	{
11252	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11253	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11254	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11255	pDbgState->fModifiedXcptBitmap = true;
11256	}
11257
11258	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11259	{
11260	pVmcsInfo->u64Cr0Mask = 0;
11261	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, 0);
11262	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11263	}
11264
11265	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11266	{
11267	pVmcsInfo->u64Cr4Mask = 0;
11268	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, 0);
11269	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11270	}
11271
11272	NOREF(pVCpu);
11273	}
11274
11275
11276	/**
11277	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11278	* re-entry next time around.
11279	*
11280	* @returns Strict VBox status code (i.e. informational status codes too).
11281	* @param pVCpu The cross context virtual CPU structure.
11282	* @param pVmxTransient The VMX-transient structure.
11283	* @param pDbgState The debug state.
11284	* @param rcStrict The return code from executing the guest using single
11285	* stepping.
11286	*/
11287	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11288	VBOXSTRICTRC rcStrict)
11289	{
11290	/*
11291	* Restore VM-exit control settings as we may not reenter this function the
11292	* next time around.
11293	*/
11294	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11295
11296	/* We reload the initial value, trigger what we can of recalculations the
11297	next time around. From the looks of things, that's all that's required atm. */
11298	if (pDbgState->fModifiedProcCtls)
11299	{
11300	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11301	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11302	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11303	AssertRC(rc2);
11304	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11305	}
11306
11307	/* We're currently the only ones messing with this one, so just restore the
11308	cached value and reload the field. */
11309	if ( pDbgState->fModifiedProcCtls2
11310	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11311	{
11312	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11313	AssertRC(rc2);
11314	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11315	}
11316
11317	/* If we've modified the exception bitmap, we restore it and trigger
11318	reloading and partial recalculation the next time around. */
11319	if (pDbgState->fModifiedXcptBitmap)
11320	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11321
11322	return rcStrict;
11323	}
11324
11325
11326	/**
11327	* Configures VM-exit controls for current DBGF and DTrace settings.
11328	*
11329	* This updates @a pDbgState and the VMCS execution control fields to reflect
11330	* the necessary VM-exits demanded by DBGF and DTrace.
11331	*
11332	* @param pVCpu The cross context virtual CPU structure.
11333	* @param pVmxTransient The VMX-transient structure. May update
11334	* fUpdatedTscOffsettingAndPreemptTimer.
11335	* @param pDbgState The debug state.
11336	*/
11337	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11338	{
11339	/*
11340	* Take down the dtrace serial number so we can spot changes.
11341	*/
11342	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11343	ASMCompilerBarrier();
11344
11345	/*
11346	* We'll rebuild most of the middle block of data members (holding the
11347	* current settings) as we go along here, so start by clearing it all.
11348	*/
11349	pDbgState->bmXcptExtra = 0;
11350	pDbgState->fCpe1Extra = 0;
11351	pDbgState->fCpe1Unwanted = 0;
11352	pDbgState->fCpe2Extra = 0;
11353	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11354	pDbgState->bmExitsToCheck[i] = 0;
11355
11356	/*
11357	* Software interrupts (INT XXh) - no idea how to trigger these...
11358	*/
11359	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11360	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11361	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11362	{
11363	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11364	}
11365
11366	/*
11367	* INT3 breakpoints - triggered by #BP exceptions.
11368	*/
11369	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11370	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11371
11372	/*
11373	* Exception bitmap and XCPT events+probes.
11374	*/
11375	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11376	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11377	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11378
11379	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11380	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11381	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11382	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11383	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11384	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11385	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11386	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11387	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11388	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11389	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11390	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11391	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11392	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11393	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11394	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11395	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11396	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11397
11398	if (pDbgState->bmXcptExtra)
11399	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11400
11401	/*
11402	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11403	*
11404	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11405	* So, when adding/changing/removing please don't forget to update it.
11406	*
11407	* Some of the macros are picking up local variables to save horizontal space,
11408	* (being able to see it in a table is the lesser evil here).
11409	*/
11410	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11411	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11412	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11413	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11414	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11415	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11416	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11417	} else do { } while (0)
11418	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11419	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11420	{ \
11421	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11422	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11423	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11424	} else do { } while (0)
11425	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11426	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11427	{ \
11428	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11429	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11430	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11431	} else do { } while (0)
11432	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11433	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11434	{ \
11435	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11436	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11437	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11438	} else do { } while (0)
11439
11440	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11441	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11442	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11443	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11444	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11445
11446	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11447	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11448	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11449	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11450	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11451	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11452	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11453	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11454	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11455	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11456	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11457	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11458	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11459	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11460	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11461	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11462	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11463	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11464	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11465	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11466	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11467	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11468	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11469	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11470	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11471	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11472	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11473	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11474	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11475	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11476	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11477	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11478	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11479	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11480	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11481	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11482
11483	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11484	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11485	{
11486	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11487	\| CPUMCTX_EXTRN_APIC_TPR);
11488	AssertRC(rc);
11489
11490	#if 0 /** @todo fix me */
11491	pDbgState->fClearCr0Mask = true;
11492	pDbgState->fClearCr4Mask = true;
11493	#endif
11494	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11495	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11496	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11497	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11498	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11499	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11500	require clearing here and in the loop if we start using it. */
11501	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11502	}
11503	else
11504	{
11505	if (pDbgState->fClearCr0Mask)
11506	{
11507	pDbgState->fClearCr0Mask = false;
11508	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11509	}
11510	if (pDbgState->fClearCr4Mask)
11511	{
11512	pDbgState->fClearCr4Mask = false;
11513	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11514	}
11515	}
11516	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11517	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11518
11519	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11520	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11521	{
11522	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11523	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11524	}
11525	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11526	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11527
11528	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11529	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11530	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11531	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11532	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11533	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11534	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11535	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11536	#if 0 /** @todo too slow, fix handler. */
11537	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11538	#endif
11539	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11540
11541	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11542	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11543	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11544	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11545	{
11546	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11547	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11548	}
11549	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11550	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11551	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11552	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11553
11554	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11555	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11556	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11557	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11558	{
11559	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11560	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11561	}
11562	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11563	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11564	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11565	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11566
11567	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11568	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11569	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11570	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11571	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11572	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11573	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11574	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11575	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11576	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11577	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11578	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11579	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11580	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11581	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11582	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11583	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11584	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11585	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11586	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11587	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11588	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11589
11590	#undef IS_EITHER_ENABLED
11591	#undef SET_ONLY_XBM_IF_EITHER_EN
11592	#undef SET_CPE1_XBM_IF_EITHER_EN
11593	#undef SET_CPEU_XBM_IF_EITHER_EN
11594	#undef SET_CPE2_XBM_IF_EITHER_EN
11595
11596	/*
11597	* Sanitize the control stuff.
11598	*/
11599	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11600	if (pDbgState->fCpe2Extra)
11601	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11602	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11603	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11604	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11605	{
11606	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11607	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11608	}
11609
11610	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11611	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11612	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11613	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11614	}
11615
11616
11617	/**
11618	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11619	* appropriate.
11620	*
11621	* The caller has checked the VM-exit against the
11622	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11623	* already, so we don't have to do that either.
11624	*
11625	* @returns Strict VBox status code (i.e. informational status codes too).
11626	* @param pVCpu The cross context virtual CPU structure.
11627	* @param pVmxTransient The VMX-transient structure.
11628	* @param uExitReason The VM-exit reason.
11629	*
11630	* @remarks The name of this function is displayed by dtrace, so keep it short
11631	* and to the point. No longer than 33 chars long, please.
11632	*/
11633	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11634	{
11635	/*
11636	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11637	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11638	*
11639	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11640	* does. Must add/change/remove both places. Same ordering, please.
11641	*
11642	* Added/removed events must also be reflected in the next section
11643	* where we dispatch dtrace events.
11644	*/
11645	bool fDtrace1 = false;
11646	bool fDtrace2 = false;
11647	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11648	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11649	uint32_t uEventArg = 0;
11650	#define SET_EXIT(a_EventSubName) \
11651	do { \
11652	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11653	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11654	} while (0)
11655	#define SET_BOTH(a_EventSubName) \
11656	do { \
11657	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11658	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11659	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11660	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11661	} while (0)
11662	switch (uExitReason)
11663	{
11664	case VMX_EXIT_MTF:
11665	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11666
11667	case VMX_EXIT_XCPT_OR_NMI:
11668	{
11669	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11670	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11671	{
11672	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11673	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11674	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11675	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11676	{
11677	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11678	{
11679	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11680	uEventArg = pVmxTransient->uExitIntErrorCode;
11681	}
11682	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11683	switch (enmEvent1)
11684	{
11685	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11686	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11687	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11688	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11689	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11690	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11691	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11692	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11693	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11694	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11695	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11696	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11697	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11698	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11699	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11700	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11701	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11702	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11703	default: break;
11704	}
11705	}
11706	else
11707	AssertFailed();
11708	break;
11709
11710	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11711	uEventArg = idxVector;
11712	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11713	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11714	break;
11715	}
11716	break;
11717	}
11718
11719	case VMX_EXIT_TRIPLE_FAULT:
11720	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11721	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11722	break;
11723	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11724	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11725	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11726	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11727	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11728
11729	/* Instruction specific VM-exits: */
11730	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11731	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11732	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11733	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11734	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11735	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11736	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11737	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11738	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11739	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11740	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11741	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11742	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11743	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11744	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11745	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11746	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11747	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11748	case VMX_EXIT_MOV_CRX:
11749	hmR0VmxReadExitQualVmcs(pVmxTransient);
11750	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11751	SET_BOTH(CRX_READ);
11752	else
11753	SET_BOTH(CRX_WRITE);
11754	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11755	break;
11756	case VMX_EXIT_MOV_DRX:
11757	hmR0VmxReadExitQualVmcs(pVmxTransient);
11758	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11759	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11760	SET_BOTH(DRX_READ);
11761	else
11762	SET_BOTH(DRX_WRITE);
11763	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11764	break;
11765	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11766	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11767	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11768	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11769	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11770	case VMX_EXIT_GDTR_IDTR_ACCESS:
11771	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11772	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11773	{
11774	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11775	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11776	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11777	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11778	}
11779	break;
11780
11781	case VMX_EXIT_LDTR_TR_ACCESS:
11782	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11783	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11784	{
11785	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11786	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11787	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11788	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11789	}
11790	break;
11791
11792	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11793	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11794	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11795	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11796	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11797	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11798	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11799	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11800	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11801	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11802	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11803
11804	/* Events that aren't relevant at this point. */
11805	case VMX_EXIT_EXT_INT:
11806	case VMX_EXIT_INT_WINDOW:
11807	case VMX_EXIT_NMI_WINDOW:
11808	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11809	case VMX_EXIT_PREEMPT_TIMER:
11810	case VMX_EXIT_IO_INSTR:
11811	break;
11812
11813	/* Errors and unexpected events. */
11814	case VMX_EXIT_INIT_SIGNAL:
11815	case VMX_EXIT_SIPI:
11816	case VMX_EXIT_IO_SMI:
11817	case VMX_EXIT_SMI:
11818	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11819	case VMX_EXIT_ERR_MSR_LOAD:
11820	case VMX_EXIT_ERR_MACHINE_CHECK:
11821	case VMX_EXIT_PML_FULL:
11822	case VMX_EXIT_VIRTUALIZED_EOI:
11823	break;
11824
11825	default:
11826	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11827	break;
11828	}
11829	#undef SET_BOTH
11830	#undef SET_EXIT
11831
11832	/*
11833	* Dtrace tracepoints go first. We do them here at once so we don't
11834	* have to copy the guest state saving and stuff a few dozen times.
11835	* Down side is that we've got to repeat the switch, though this time
11836	* we use enmEvent since the probes are a subset of what DBGF does.
11837	*/
11838	if (fDtrace1 \|\| fDtrace2)
11839	{
11840	hmR0VmxReadExitQualVmcs(pVmxTransient);
11841	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11842	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11843	switch (enmEvent1)
11844	{
11845	/** @todo consider which extra parameters would be helpful for each probe. */
11846	case DBGFEVENT_END: break;
11847	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11848	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11849	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11850	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11851	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11852	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11853	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11854	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11855	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11856	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11857	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11858	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11859	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11860	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11861	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11862	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11863	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11864	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11865	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11866	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11867	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11868	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11869	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11870	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11871	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11872	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11873	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11874	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11875	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11876	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11877	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11878	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11879	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11880	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11881	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11882	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11883	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11884	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11885	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11886	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11887	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11888	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11889	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11890	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11891	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11892	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11893	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11894	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11895	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11896	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11897	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11898	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11899	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11900	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11901	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11902	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11903	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11904	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11905	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11906	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11907	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11908	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11909	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11910	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11911	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11912	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11913	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11914	}
11915	switch (enmEvent2)
11916	{
11917	/** @todo consider which extra parameters would be helpful for each probe. */
11918	case DBGFEVENT_END: break;
11919	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11920	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11921	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11922	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11923	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11924	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11925	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11926	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11927	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11928	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11929	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11930	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11931	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11932	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11933	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11934	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11935	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11936	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11937	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11938	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11939	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11940	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11941	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11942	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11943	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11944	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11945	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11946	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11947	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11948	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11949	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11950	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11951	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11952	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11953	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11954	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11955	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11956	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11957	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11958	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11959	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11960	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11961	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11962	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11963	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11964	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11965	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11966	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11967	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11968	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11969	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11970	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11971	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11972	}
11973	}
11974
11975	/*
11976	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11977	* the DBGF call will do a full check).
11978	*
11979	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11980	* Note! If we have to events, we prioritize the first, i.e. the instruction
11981	* one, in order to avoid event nesting.
11982	*/
11983	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11984	if ( enmEvent1 != DBGFEVENT_END
11985	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11986	{
11987	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11988	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11989	if (rcStrict != VINF_SUCCESS)
11990	return rcStrict;
11991	}
11992	else if ( enmEvent2 != DBGFEVENT_END
11993	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11994	{
11995	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11996	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11997	if (rcStrict != VINF_SUCCESS)
11998	return rcStrict;
11999	}
12000
12001	return VINF_SUCCESS;
12002	}
12003
12004
12005	/**
12006	* Single-stepping VM-exit filtering.
12007	*
12008	* This is preprocessing the VM-exits and deciding whether we've gotten far
12009	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12010	* handling is performed.
12011	*
12012	* @returns Strict VBox status code (i.e. informational status codes too).
12013	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12014	* @param pVmxTransient The VMX-transient structure.
12015	* @param pDbgState The debug state.
12016	*/
12017	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12018	{
12019	/*
12020	* Expensive (saves context) generic dtrace VM-exit probe.
12021	*/
12022	uint32_t const uExitReason = pVmxTransient->uExitReason;
12023	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12024	{ /* more likely */ }
12025	else
12026	{
12027	hmR0VmxReadExitQualVmcs(pVmxTransient);
12028	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12029	AssertRC(rc);
12030	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12031	}
12032
12033	/*
12034	* Check for host NMI, just to get that out of the way.
12035	*/
12036	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12037	{ /* normally likely */ }
12038	else
12039	{
12040	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12041	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12042	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12043	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12044	}
12045
12046	/*
12047	* Check for single stepping event if we're stepping.
12048	*/
12049	if (pVCpu->hm.s.fSingleInstruction)
12050	{
12051	switch (uExitReason)
12052	{
12053	case VMX_EXIT_MTF:
12054	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12055
12056	/* Various events: */
12057	case VMX_EXIT_XCPT_OR_NMI:
12058	case VMX_EXIT_EXT_INT:
12059	case VMX_EXIT_TRIPLE_FAULT:
12060	case VMX_EXIT_INT_WINDOW:
12061	case VMX_EXIT_NMI_WINDOW:
12062	case VMX_EXIT_TASK_SWITCH:
12063	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12064	case VMX_EXIT_APIC_ACCESS:
12065	case VMX_EXIT_EPT_VIOLATION:
12066	case VMX_EXIT_EPT_MISCONFIG:
12067	case VMX_EXIT_PREEMPT_TIMER:
12068
12069	/* Instruction specific VM-exits: */
12070	case VMX_EXIT_CPUID:
12071	case VMX_EXIT_GETSEC:
12072	case VMX_EXIT_HLT:
12073	case VMX_EXIT_INVD:
12074	case VMX_EXIT_INVLPG:
12075	case VMX_EXIT_RDPMC:
12076	case VMX_EXIT_RDTSC:
12077	case VMX_EXIT_RSM:
12078	case VMX_EXIT_VMCALL:
12079	case VMX_EXIT_VMCLEAR:
12080	case VMX_EXIT_VMLAUNCH:
12081	case VMX_EXIT_VMPTRLD:
12082	case VMX_EXIT_VMPTRST:
12083	case VMX_EXIT_VMREAD:
12084	case VMX_EXIT_VMRESUME:
12085	case VMX_EXIT_VMWRITE:
12086	case VMX_EXIT_VMXOFF:
12087	case VMX_EXIT_VMXON:
12088	case VMX_EXIT_MOV_CRX:
12089	case VMX_EXIT_MOV_DRX:
12090	case VMX_EXIT_IO_INSTR:
12091	case VMX_EXIT_RDMSR:
12092	case VMX_EXIT_WRMSR:
12093	case VMX_EXIT_MWAIT:
12094	case VMX_EXIT_MONITOR:
12095	case VMX_EXIT_PAUSE:
12096	case VMX_EXIT_GDTR_IDTR_ACCESS:
12097	case VMX_EXIT_LDTR_TR_ACCESS:
12098	case VMX_EXIT_INVEPT:
12099	case VMX_EXIT_RDTSCP:
12100	case VMX_EXIT_INVVPID:
12101	case VMX_EXIT_WBINVD:
12102	case VMX_EXIT_XSETBV:
12103	case VMX_EXIT_RDRAND:
12104	case VMX_EXIT_INVPCID:
12105	case VMX_EXIT_VMFUNC:
12106	case VMX_EXIT_RDSEED:
12107	case VMX_EXIT_XSAVES:
12108	case VMX_EXIT_XRSTORS:
12109	{
12110	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12111	AssertRCReturn(rc, rc);
12112	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12113	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12114	return VINF_EM_DBG_STEPPED;
12115	break;
12116	}
12117
12118	/* Errors and unexpected events: */
12119	case VMX_EXIT_INIT_SIGNAL:
12120	case VMX_EXIT_SIPI:
12121	case VMX_EXIT_IO_SMI:
12122	case VMX_EXIT_SMI:
12123	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12124	case VMX_EXIT_ERR_MSR_LOAD:
12125	case VMX_EXIT_ERR_MACHINE_CHECK:
12126	case VMX_EXIT_PML_FULL:
12127	case VMX_EXIT_VIRTUALIZED_EOI:
12128	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12129	break;
12130
12131	default:
12132	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12133	break;
12134	}
12135	}
12136
12137	/*
12138	* Check for debugger event breakpoints and dtrace probes.
12139	*/
12140	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12141	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12142	{
12143	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12144	if (rcStrict != VINF_SUCCESS)
12145	return rcStrict;
12146	}
12147
12148	/*
12149	* Normal processing.
12150	*/
12151	#ifdef HMVMX_USE_FUNCTION_TABLE
12152	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12153	#else
12154	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12155	#endif
12156	}
12157
12158
12159	/**
12160	* Single steps guest code using hardware-assisted VMX.
12161	*
12162	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12163	* but single-stepping through the hypervisor debugger.
12164	*
12165	* @returns Strict VBox status code (i.e. informational status codes too).
12166	* @param pVCpu The cross context virtual CPU structure.
12167	* @param pcLoops Pointer to the number of executed loops.
12168	*
12169	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12170	*/
12171	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
12172	{
12173	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12174	Assert(pcLoops);
12175	Assert(*pcLoops <= cMaxResumeLoops);
12176
12177	VMXTRANSIENT VmxTransient;
12178	RT_ZERO(VmxTransient);
12179	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12180
12181	/* Set HMCPU indicators. */
12182	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12183	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12184	pVCpu->hm.s.fDebugWantRdTscExit = false;
12185	pVCpu->hm.s.fUsingDebugLoop = true;
12186
12187	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12188	VMXRUNDBGSTATE DbgState;
12189	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12190	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12191
12192	/*
12193	* The loop.
12194	*/
12195	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12196	for (;;)
12197	{
12198	Assert(!HMR0SuspendPending());
12199	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12200	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12201	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12202
12203	/* Set up VM-execution controls the next two can respond to. */
12204	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12205
12206	/*
12207	* Preparatory work for running guest code, this may force us to
12208	* return to ring-3.
12209	*
12210	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12211	*/
12212	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12213	if (rcStrict != VINF_SUCCESS)
12214	break;
12215
12216	/* Interrupts are disabled at this point! */
12217	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12218
12219	/* Override any obnoxious code in the above two calls. */
12220	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12221
12222	/*
12223	* Finally execute the guest.
12224	*/
12225	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12226
12227	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12228	/* Interrupts are re-enabled at this point! */
12229
12230	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12231	if (RT_SUCCESS(rcRun))
12232	{ /* very likely */ }
12233	else
12234	{
12235	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12236	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12237	return rcRun;
12238	}
12239
12240	/* Profile the VM-exit. */
12241	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12242	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12243	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12244	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12245	HMVMX_START_EXIT_DISPATCH_PROF();
12246
12247	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12248
12249	/*
12250	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12251	*/
12252	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12253	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12254	if (rcStrict != VINF_SUCCESS)
12255	break;
12256	if (++(*pcLoops) > cMaxResumeLoops)
12257	{
12258	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12259	rcStrict = VINF_EM_RAW_INTERRUPT;
12260	break;
12261	}
12262
12263	/*
12264	* Stepping: Did the RIP change, if so, consider it a single step.
12265	* Otherwise, make sure one of the TFs gets set.
12266	*/
12267	if (fStepping)
12268	{
12269	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12270	AssertRC(rc);
12271	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12272	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12273	{
12274	rcStrict = VINF_EM_DBG_STEPPED;
12275	break;
12276	}
12277	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12278	}
12279
12280	/*
12281	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12282	*/
12283	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12284	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12285	}
12286
12287	/*
12288	* Clear the X86_EFL_TF if necessary.
12289	*/
12290	if (pVCpu->hm.s.fClearTrapFlag)
12291	{
12292	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12293	AssertRC(rc);
12294	pVCpu->hm.s.fClearTrapFlag = false;
12295	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12296	}
12297	/** @todo there seems to be issues with the resume flag when the monitor trap
12298	* flag is pending without being used. Seen early in bios init when
12299	* accessing APIC page in protected mode. */
12300
12301	/*
12302	* Restore VM-exit control settings as we may not re-enter this function the
12303	* next time around.
12304	*/
12305	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12306
12307	/* Restore HMCPU indicators. */
12308	pVCpu->hm.s.fUsingDebugLoop = false;
12309	pVCpu->hm.s.fDebugWantRdTscExit = false;
12310	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12311
12312	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12313	return rcStrict;
12314	}
12315
12316
12317	/** @} */
12318
12319
12320	/**
12321	* Checks if any expensive dtrace probes are enabled and we should go to the
12322	* debug loop.
12323	*
12324	* @returns true if we should use debug loop, false if not.
12325	*/
12326	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12327	{
12328	/* It's probably faster to OR the raw 32-bit counter variables together.
12329	Since the variables are in an array and the probes are next to one
12330	another (more or less), we have good locality. So, better read
12331	eight-nine cache lines ever time and only have one conditional, than
12332	128+ conditionals, right? */
12333	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12334	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12335	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12336	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12337	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12338	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12339	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12340	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12341	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12342	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12343	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12344	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12345	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12346	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12347	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12348	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12349	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12350	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12351	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12352	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12353	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12354	) != 0
12355	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12356	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12357	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12358	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12359	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12360	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12361	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12362	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12363	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12364	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12365	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12366	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12367	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12368	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12369	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12370	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12371	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12372	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12373	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12374	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12375	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12376	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12377	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12378	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12379	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12380	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12381	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12382	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12383	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12384	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12385	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12386	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12387	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12388	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12389	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12390	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12391	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12392	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12393	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12394	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12395	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12396	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12397	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12398	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12399	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12400	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12401	) != 0
12402	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12403	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12404	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12405	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12406	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12407	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12408	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12409	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12410	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12411	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12412	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12413	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12414	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12415	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12416	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12417	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12418	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12419	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12420	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12421	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12422	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12423	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12424	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12425	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12426	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12427	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12428	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12429	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12430	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12431	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12432	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12433	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12434	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12435	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12436	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12437	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12438	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12439	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12440	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12441	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12442	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12443	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12444	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12445	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12446	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12447	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12448	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12449	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12450	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12451	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12452	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12453	) != 0;
12454	}
12455
12456
12457	/**
12458	* Runs the guest using hardware-assisted VMX.
12459	*
12460	* @returns Strict VBox status code (i.e. informational status codes too).
12461	* @param pVCpu The cross context virtual CPU structure.
12462	*/
12463	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu)
12464	{
12465	AssertPtr(pVCpu);
12466	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12467	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12468	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12469	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12470
12471	VBOXSTRICTRC rcStrict;
12472	uint32_t cLoops = 0;
12473	for (;;)
12474	{
12475	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12476	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12477	#else
12478	NOREF(pCtx);
12479	bool const fInNestedGuestMode = false;
12480	#endif
12481	if (!fInNestedGuestMode)
12482	{
12483	if ( !pVCpu->hm.s.fUseDebugLoop
12484	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12485	&& !DBGFIsStepping(pVCpu)
12486	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12487	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12488	else
12489	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12490	}
12491	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12492	else
12493	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12494
12495	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12496	{
12497	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12498	continue;
12499	}
12500	if (rcStrict == VINF_VMX_VMEXIT)
12501	{
12502	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12503	continue;
12504	}
12505	#endif
12506	break;
12507	}
12508
12509	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12510	switch (rcLoop)
12511	{
12512	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12513	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12514	}
12515
12516	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12517	if (RT_FAILURE(rc2))
12518	{
12519	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12520	rcStrict = rc2;
12521	}
12522	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12523	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12524	return rcStrict;
12525	}
12526
12527
12528	#ifndef HMVMX_USE_FUNCTION_TABLE
12529	/**
12530	* Handles a guest VM-exit from hardware-assisted VMX execution.
12531	*
12532	* @returns Strict VBox status code (i.e. informational status codes too).
12533	* @param pVCpu The cross context virtual CPU structure.
12534	* @param pVmxTransient The VMX-transient structure.
12535	*/
12536	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12537	{
12538	#ifdef DEBUG_ramshankar
12539	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12540	do { \
12541	if (a_fSave != 0) \
12542	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12543	VBOXSTRICTRC rcStrict = a_CallExpr; \
12544	if (a_fSave != 0) \
12545	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12546	return rcStrict; \
12547	} while (0)
12548	#else
12549	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12550	#endif
12551	uint32_t const uExitReason = pVmxTransient->uExitReason;
12552	switch (uExitReason)
12553	{
12554	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12555	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12556	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12557	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12558	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12559	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12560	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12561	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12562	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12563	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12564	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12565	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12566	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12567	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12568	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12569	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12570	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12571	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12572	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12573	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12574	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12575	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12576	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12577	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12578	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12579	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12580	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12581	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12582	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12583	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12584	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12585	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12586	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12587	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12588	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12589	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12590	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12591	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12592	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12593	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12594	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12595	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12596	#else
12597	case VMX_EXIT_VMCLEAR:
12598	case VMX_EXIT_VMLAUNCH:
12599	case VMX_EXIT_VMPTRLD:
12600	case VMX_EXIT_VMPTRST:
12601	case VMX_EXIT_VMREAD:
12602	case VMX_EXIT_VMRESUME:
12603	case VMX_EXIT_VMWRITE:
12604	case VMX_EXIT_VMXOFF:
12605	case VMX_EXIT_VMXON:
12606	case VMX_EXIT_INVVPID:
12607	case VMX_EXIT_INVEPT:
12608	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12609	#endif
12610
12611	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12612	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12613	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12614
12615	case VMX_EXIT_INIT_SIGNAL:
12616	case VMX_EXIT_SIPI:
12617	case VMX_EXIT_IO_SMI:
12618	case VMX_EXIT_SMI:
12619	case VMX_EXIT_ERR_MSR_LOAD:
12620	case VMX_EXIT_ERR_MACHINE_CHECK:
12621	case VMX_EXIT_PML_FULL:
12622	case VMX_EXIT_VIRTUALIZED_EOI:
12623	case VMX_EXIT_GDTR_IDTR_ACCESS:
12624	case VMX_EXIT_LDTR_TR_ACCESS:
12625	case VMX_EXIT_APIC_WRITE:
12626	case VMX_EXIT_RDRAND:
12627	case VMX_EXIT_RSM:
12628	case VMX_EXIT_VMFUNC:
12629	case VMX_EXIT_ENCLS:
12630	case VMX_EXIT_RDSEED:
12631	case VMX_EXIT_XSAVES:
12632	case VMX_EXIT_XRSTORS:
12633	case VMX_EXIT_UMWAIT:
12634	case VMX_EXIT_TPAUSE:
12635	default:
12636	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12637	}
12638	#undef VMEXIT_CALL_RET
12639	}
12640	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12641
12642
12643	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12644	/**
12645	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12646	*
12647	* @returns Strict VBox status code (i.e. informational status codes too).
12648	* @param pVCpu The cross context virtual CPU structure.
12649	* @param pVmxTransient The VMX-transient structure.
12650	*/
12651	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12652	{
12653	/** @todo NSTVMX: Remove after debugging page-fault issue. */
12654	#ifdef DEBUG_ramshankar
12655	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12656	Log4Func(("cs:rip=%#04x:%#RX64 rsp=%#RX64 eflags=%#RX32 cr3=%#RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12657	pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.cr3));
12658	#endif
12659
12660	uint32_t const uExitReason = pVmxTransient->uExitReason;
12661	switch (uExitReason)
12662	{
12663	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12664	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12665	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12666	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12667	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12668
12669	/*
12670	* We shouldn't direct host physical interrupts to the nested-guest.
12671	*/
12672	case VMX_EXIT_EXT_INT:
12673	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12674
12675	/*
12676	* Instructions that cause VM-exits unconditionally or the condition is
12677	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12678	* happens, it's guaranteed to be a nested-guest VM-exit).
12679	*
12680	* - Provides VM-exit instruction length ONLY.
12681	*/
12682	case VMX_EXIT_CPUID: /* Unconditional. */
12683	case VMX_EXIT_VMCALL:
12684	case VMX_EXIT_GETSEC:
12685	case VMX_EXIT_INVD:
12686	case VMX_EXIT_XSETBV:
12687	case VMX_EXIT_VMLAUNCH:
12688	case VMX_EXIT_VMRESUME:
12689	case VMX_EXIT_VMXOFF:
12690	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
12691	case VMX_EXIT_VMFUNC:
12692	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12693
12694	/*
12695	* Instructions that cause VM-exits unconditionally or the condition is
12696	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12697	* happens, it's guaranteed to be a nested-guest VM-exit).
12698	*
12699	* - Provides VM-exit instruction length.
12700	* - Provides VM-exit information.
12701	* - Optionally provides Exit qualification.
12702	*
12703	* Since Exit qualification is 0 for all VM-exits where it is not
12704	* applicable, reading and passing it to the guest should produce
12705	* defined behavior.
12706	*
12707	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12708	*/
12709	case VMX_EXIT_INVEPT: /* Unconditional. */
12710	case VMX_EXIT_INVVPID:
12711	case VMX_EXIT_VMCLEAR:
12712	case VMX_EXIT_VMPTRLD:
12713	case VMX_EXIT_VMPTRST:
12714	case VMX_EXIT_VMXON:
12715	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
12716	case VMX_EXIT_LDTR_TR_ACCESS:
12717	case VMX_EXIT_RDRAND:
12718	case VMX_EXIT_RDSEED:
12719	case VMX_EXIT_XSAVES:
12720	case VMX_EXIT_XRSTORS:
12721	case VMX_EXIT_UMWAIT:
12722	case VMX_EXIT_TPAUSE:
12723	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12724
12725	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12726	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12727	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12728	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12729	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12730	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12731	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12732	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12733	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12734	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12735	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12736	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12737	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12738	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12739	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12740	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12741	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12742	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12743	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12744
12745	case VMX_EXIT_PREEMPT_TIMER:
12746	{
12747	/** @todo NSTVMX: Preempt timer. */
12748	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12749	}
12750
12751	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12752	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12753
12754	case VMX_EXIT_VMREAD:
12755	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12756
12757	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12758	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12759
12760	case VMX_EXIT_INIT_SIGNAL:
12761	case VMX_EXIT_SIPI:
12762	case VMX_EXIT_IO_SMI:
12763	case VMX_EXIT_SMI:
12764	case VMX_EXIT_ERR_MSR_LOAD:
12765	case VMX_EXIT_ERR_MACHINE_CHECK:
12766	case VMX_EXIT_PML_FULL:
12767	case VMX_EXIT_RSM:
12768	default:
12769	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12770	}
12771	}
12772	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12773
12774
12775	/** @name VM-exit helpers.
12776	* @{
12777	*/
12778	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12779	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12780	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12781
12782	/** Macro for VM-exits called unexpectedly. */
12783	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
12784	do { \
12785	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
12786	return VERR_VMX_UNEXPECTED_EXIT; \
12787	} while (0)
12788
12789	#ifdef VBOX_STRICT
12790	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12791	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12792	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12793
12794	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12795	do { \
12796	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12797	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12798	} while (0)
12799
12800	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12801	do { \
12802	AssertPtr((a_pVCpu)); \
12803	AssertPtr((a_pVmxTransient)); \
12804	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12805	Assert((a_pVmxTransient)->pVmcsInfo); \
12806	Assert(ASMIntAreEnabled()); \
12807	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12808	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12809	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
12810	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12811	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12812	HMVMX_ASSERT_PREEMPT_CPUID(); \
12813	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12814	} while (0)
12815
12816	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12817	do { \
12818	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12819	Assert((a_pVmxTransient)->fIsNestedGuest); \
12820	} while (0)
12821
12822	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12823	do { \
12824	Log4Func(("\n")); \
12825	} while (0)
12826	#else
12827	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12828	do { \
12829	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12830	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12831	} while (0)
12832
12833	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12834	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12835
12836	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12837	#endif
12838
12839	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12840	/** Macro that does the necessary privilege checks and intercepted VM-exits for
12841	* guests that attempted to execute a VMX instruction. */
12842	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
12843	do \
12844	{ \
12845	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
12846	if (rcStrictTmp == VINF_SUCCESS) \
12847	{ /* likely */ } \
12848	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12849	{ \
12850	Assert((a_pVCpu)->hm.s.Event.fPending); \
12851	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
12852	return VINF_SUCCESS; \
12853	} \
12854	else \
12855	{ \
12856	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
12857	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
12858	} \
12859	} while (0)
12860
12861	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
12862	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
12863	do \
12864	{ \
12865	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
12866	(a_pGCPtrEffAddr)); \
12867	if (rcStrictTmp == VINF_SUCCESS) \
12868	{ /* likely */ } \
12869	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12870	{ \
12871	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
12872	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
12873	NOREF(uXcptTmp); \
12874	return VINF_SUCCESS; \
12875	} \
12876	else \
12877	{ \
12878	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
12879	return rcStrictTmp; \
12880	} \
12881	} while (0)
12882	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12883
12884
12885	/**
12886	* Advances the guest RIP by the specified number of bytes.
12887	*
12888	* @param pVCpu The cross context virtual CPU structure.
12889	* @param cbInstr Number of bytes to advance the RIP by.
12890	*
12891	* @remarks No-long-jump zone!!!
12892	*/
12893	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
12894	{
12895	/* Advance the RIP. */
12896	pVCpu->cpum.GstCtx.rip += cbInstr;
12897	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12898
12899	/* Update interrupt inhibition. */
12900	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12901	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12902	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12903	}
12904
12905
12906	/**
12907	* Advances the guest RIP after reading it from the VMCS.
12908	*
12909	* @returns VBox status code, no informational status codes.
12910	* @param pVCpu The cross context virtual CPU structure.
12911	* @param pVmxTransient The VMX-transient structure.
12912	*
12913	* @remarks No-long-jump zone!!!
12914	*/
12915	static int hmR0VmxAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12916	{
12917	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12918	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12919	AssertRCReturn(rc, rc);
12920
12921	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
12922	return VINF_SUCCESS;
12923	}
12924
12925
12926	/**
12927	* Handle a condition that occurred while delivering an event through the guest or
12928	* nested-guest IDT.
12929	*
12930	* @returns Strict VBox status code (i.e. informational status codes too).
12931	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12932	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12933	* to continue execution of the guest which will delivery the \#DF.
12934	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12935	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12936	*
12937	* @param pVCpu The cross context virtual CPU structure.
12938	* @param pVmxTransient The VMX-transient structure.
12939	*
12940	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
12941	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
12942	* is due to an EPT violation, PML full or SPP-related event.
12943	*
12944	* @remarks No-long-jump zone!!!
12945	*/
12946	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12947	{
12948	Assert(!pVCpu->hm.s.Event.fPending);
12949	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
12950	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
12951	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
12952	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
12953	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
12954
12955	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12956	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12957	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
12958	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12959	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
12960	{
12961	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
12962	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
12963
12964	/*
12965	* If the event was a software interrupt (generated with INT n) or a software exception
12966	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12967	* can handle the VM-exit and continue guest execution which will re-execute the
12968	* instruction rather than re-injecting the exception, as that can cause premature
12969	* trips to ring-3 before injection and involve TRPM which currently has no way of
12970	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12971	* the problem).
12972	*/
12973	IEMXCPTRAISE enmRaise;
12974	IEMXCPTRAISEINFO fRaiseInfo;
12975	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12976	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12977	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12978	{
12979	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12980	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12981	}
12982	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
12983	{
12984	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
12985	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12986	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
12987
12988	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12989	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12990
12991	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12992
12993	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12994	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12995	{
12996	pVmxTransient->fVectoringPF = true;
12997	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12998	}
12999	}
13000	else
13001	{
13002	/*
13003	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13004	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13005	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13006	*/
13007	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13008	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13009	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13010	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13011	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13012	}
13013
13014	/*
13015	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13016	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13017	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13018	* subsequent VM-entry would fail, see @bugref{7445}.
13019	*
13020	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13021	*/
13022	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13023	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
13024	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13025	&& CPUMIsGuestNmiBlocking(pVCpu))
13026	{
13027	CPUMSetGuestNmiBlocking(pVCpu, false);
13028	}
13029
13030	switch (enmRaise)
13031	{
13032	case IEMXCPTRAISE_CURRENT_XCPT:
13033	{
13034	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
13035	Assert(rcStrict == VINF_SUCCESS);
13036	break;
13037	}
13038
13039	case IEMXCPTRAISE_PREV_EVENT:
13040	{
13041	uint32_t u32ErrCode;
13042	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13043	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13044	else
13045	u32ErrCode = 0;
13046
13047	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13048	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13049	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13050	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13051
13052	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13053	pVCpu->hm.s.Event.u32ErrCode));
13054	Assert(rcStrict == VINF_SUCCESS);
13055	break;
13056	}
13057
13058	case IEMXCPTRAISE_REEXEC_INSTR:
13059	Assert(rcStrict == VINF_SUCCESS);
13060	break;
13061
13062	case IEMXCPTRAISE_DOUBLE_FAULT:
13063	{
13064	/*
13065	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13066	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13067	*/
13068	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13069	{
13070	pVmxTransient->fVectoringDoublePF = true;
13071	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13072	pVCpu->cpum.GstCtx.cr2));
13073	rcStrict = VINF_SUCCESS;
13074	}
13075	else
13076	{
13077	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13078	hmR0VmxSetPendingXcptDF(pVCpu);
13079	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13080	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13081	rcStrict = VINF_HM_DOUBLE_FAULT;
13082	}
13083	break;
13084	}
13085
13086	case IEMXCPTRAISE_TRIPLE_FAULT:
13087	{
13088	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13089	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13090	rcStrict = VINF_EM_RESET;
13091	break;
13092	}
13093
13094	case IEMXCPTRAISE_CPU_HANG:
13095	{
13096	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13097	rcStrict = VERR_EM_GUEST_CPU_HANG;
13098	break;
13099	}
13100
13101	default:
13102	{
13103	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13104	rcStrict = VERR_VMX_IPE_2;
13105	break;
13106	}
13107	}
13108	}
13109	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13110	&& !CPUMIsGuestNmiBlocking(pVCpu))
13111	{
13112	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13113	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13114	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13115	{
13116	/*
13117	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13118	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13119	* that NMIs remain blocked until the IRET execution is completed.
13120	*
13121	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13122	*/
13123	CPUMSetGuestNmiBlocking(pVCpu, true);
13124	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13125	}
13126	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13127	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13128	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13129	{
13130	/*
13131	* Execution of IRET caused an EPT violation, page-modification log-full event or
13132	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13133	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13134	* that NMIs remain blocked until the IRET execution is completed.
13135	*
13136	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13137	*/
13138	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13139	{
13140	CPUMSetGuestNmiBlocking(pVCpu, true);
13141	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13142	}
13143	}
13144	}
13145
13146	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13147	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13148	return rcStrict;
13149	}
13150
13151
13152	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13153	/**
13154	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13155	* guest attempting to execute a VMX instruction.
13156	*
13157	* @returns Strict VBox status code (i.e. informational status codes too).
13158	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13159	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13160	*
13161	* @param pVCpu The cross context virtual CPU structure.
13162	* @param uExitReason The VM-exit reason.
13163	*
13164	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13165	* @remarks No-long-jump zone!!!
13166	*/
13167	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
13168	{
13169	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13170	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13171
13172	/*
13173	* The physical CPU would have already checked the CPU mode/code segment.
13174	* We shall just assert here for paranoia.
13175	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13176	*/
13177	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13178	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13179	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13180
13181	if (uExitReason == VMX_EXIT_VMXON)
13182	{
13183	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13184
13185	/*
13186	* We check CR4.VMXE because it is required to be always set while in VMX operation
13187	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13188	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13189	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13190	*/
13191	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13192	{
13193	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13194	hmR0VmxSetPendingXcptUD(pVCpu);
13195	return VINF_HM_PENDING_XCPT;
13196	}
13197	}
13198	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13199	{
13200	/*
13201	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13202	* (other than VMXON), we need to raise a #UD.
13203	*/
13204	Log4Func(("Not in VMX root mode -> #UD\n"));
13205	hmR0VmxSetPendingXcptUD(pVCpu);
13206	return VINF_HM_PENDING_XCPT;
13207	}
13208
13209	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13210	return VINF_SUCCESS;
13211	}
13212
13213
13214	/**
13215	* Decodes the memory operand of an instruction that caused a VM-exit.
13216	*
13217	* The Exit qualification field provides the displacement field for memory
13218	* operand instructions, if any.
13219	*
13220	* @returns Strict VBox status code (i.e. informational status codes too).
13221	* @retval VINF_SUCCESS if the operand was successfully decoded.
13222	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13223	* operand.
13224	* @param pVCpu The cross context virtual CPU structure.
13225	* @param uExitInstrInfo The VM-exit instruction information field.
13226	* @param enmMemAccess The memory operand's access type (read or write).
13227	* @param GCPtrDisp The instruction displacement field, if any. For
13228	* RIP-relative addressing pass RIP + displacement here.
13229	* @param pGCPtrMem Where to store the effective destination memory address.
13230	*
13231	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13232	* virtual-8086 mode hence skips those checks while verifying if the
13233	* segment is valid.
13234	*/
13235	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13236	PRTGCPTR pGCPtrMem)
13237	{
13238	Assert(pGCPtrMem);
13239	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13240	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13241	\| CPUMCTX_EXTRN_CR0);
13242
13243	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13244	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13245	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13246
13247	VMXEXITINSTRINFO ExitInstrInfo;
13248	ExitInstrInfo.u = uExitInstrInfo;
13249	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13250	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13251	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13252	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13253	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13254	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13255	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13256	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13257	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13258
13259	/*
13260	* Validate instruction information.
13261	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13262	*/
13263	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13264	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13265	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13266	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13267	AssertLogRelMsgReturn(fIsMemOperand,
13268	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13269
13270	/*
13271	* Compute the complete effective address.
13272	*
13273	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13274	* See AMD spec. 4.5.2 "Segment Registers".
13275	*/
13276	RTGCPTR GCPtrMem = GCPtrDisp;
13277	if (fBaseRegValid)
13278	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13279	if (fIdxRegValid)
13280	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13281
13282	RTGCPTR const GCPtrOff = GCPtrMem;
13283	if ( !fIsLongMode
13284	\|\| iSegReg >= X86_SREG_FS)
13285	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13286	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13287
13288	/*
13289	* Validate effective address.
13290	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13291	*/
13292	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13293	Assert(cbAccess > 0);
13294	if (fIsLongMode)
13295	{
13296	if (X86_IS_CANONICAL(GCPtrMem))
13297	{
13298	*pGCPtrMem = GCPtrMem;
13299	return VINF_SUCCESS;
13300	}
13301
13302	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13303	* "Data Limit Checks in 64-bit Mode". */
13304	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13305	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13306	return VINF_HM_PENDING_XCPT;
13307	}
13308
13309	/*
13310	* This is a watered down version of iemMemApplySegment().
13311	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13312	* and segment CPL/DPL checks are skipped.
13313	*/
13314	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13315	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13316	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13317
13318	/* Check if the segment is present and usable. */
13319	if ( pSel->Attr.n.u1Present
13320	&& !pSel->Attr.n.u1Unusable)
13321	{
13322	Assert(pSel->Attr.n.u1DescType);
13323	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13324	{
13325	/* Check permissions for the data segment. */
13326	if ( enmMemAccess == VMXMEMACCESS_WRITE
13327	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13328	{
13329	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13330	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13331	return VINF_HM_PENDING_XCPT;
13332	}
13333
13334	/* Check limits if it's a normal data segment. */
13335	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13336	{
13337	if ( GCPtrFirst32 > pSel->u32Limit
13338	\|\| GCPtrLast32 > pSel->u32Limit)
13339	{
13340	Log4Func(("Data segment limit exceeded. "
13341	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13342	GCPtrLast32, pSel->u32Limit));
13343	if (iSegReg == X86_SREG_SS)
13344	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13345	else
13346	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13347	return VINF_HM_PENDING_XCPT;
13348	}
13349	}
13350	else
13351	{
13352	/* Check limits if it's an expand-down data segment.
13353	Note! The upper boundary is defined by the B bit, not the G bit! */
13354	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13355	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13356	{
13357	Log4Func(("Expand-down data segment limit exceeded. "
13358	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13359	GCPtrLast32, pSel->u32Limit));
13360	if (iSegReg == X86_SREG_SS)
13361	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13362	else
13363	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13364	return VINF_HM_PENDING_XCPT;
13365	}
13366	}
13367	}
13368	else
13369	{
13370	/* Check permissions for the code segment. */
13371	if ( enmMemAccess == VMXMEMACCESS_WRITE
13372	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13373	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13374	{
13375	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13376	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13377	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13378	return VINF_HM_PENDING_XCPT;
13379	}
13380
13381	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13382	if ( GCPtrFirst32 > pSel->u32Limit
13383	\|\| GCPtrLast32 > pSel->u32Limit)
13384	{
13385	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13386	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13387	if (iSegReg == X86_SREG_SS)
13388	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13389	else
13390	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13391	return VINF_HM_PENDING_XCPT;
13392	}
13393	}
13394	}
13395	else
13396	{
13397	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13398	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13399	return VINF_HM_PENDING_XCPT;
13400	}
13401
13402	*pGCPtrMem = GCPtrMem;
13403	return VINF_SUCCESS;
13404	}
13405	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13406
13407
13408	/**
13409	* VM-exit helper for LMSW.
13410	*/
13411	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13412	{
13413	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13414	AssertRCReturn(rc, rc);
13415
13416	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13417	AssertMsg( rcStrict == VINF_SUCCESS
13418	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13419
13420	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13421	if (rcStrict == VINF_IEM_RAISED_XCPT)
13422	{
13423	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13424	rcStrict = VINF_SUCCESS;
13425	}
13426
13427	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13428	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13429	return rcStrict;
13430	}
13431
13432
13433	/**
13434	* VM-exit helper for CLTS.
13435	*/
13436	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13437	{
13438	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13439	AssertRCReturn(rc, rc);
13440
13441	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13442	AssertMsg( rcStrict == VINF_SUCCESS
13443	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13444
13445	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13446	if (rcStrict == VINF_IEM_RAISED_XCPT)
13447	{
13448	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13449	rcStrict = VINF_SUCCESS;
13450	}
13451
13452	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13453	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13454	return rcStrict;
13455	}
13456
13457
13458	/**
13459	* VM-exit helper for MOV from CRx (CRx read).
13460	*/
13461	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13462	{
13463	Assert(iCrReg < 16);
13464	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13465
13466	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13467	AssertRCReturn(rc, rc);
13468
13469	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13470	AssertMsg( rcStrict == VINF_SUCCESS
13471	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13472
13473	if (iGReg == X86_GREG_xSP)
13474	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13475	else
13476	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13477	#ifdef VBOX_WITH_STATISTICS
13478	switch (iCrReg)
13479	{
13480	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13481	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13482	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13483	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13484	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13485	}
13486	#endif
13487	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13488	return rcStrict;
13489	}
13490
13491
13492	/**
13493	* VM-exit helper for MOV to CRx (CRx write).
13494	*/
13495	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13496	{
13497	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13498	AssertRCReturn(rc, rc);
13499
13500	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13501	AssertMsg( rcStrict == VINF_SUCCESS
13502	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13503	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13504
13505	switch (iCrReg)
13506	{
13507	case 0:
13508	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
13509	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13510	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13511	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13512	break;
13513
13514	case 2:
13515	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13516	/* Nothing to do here, CR2 it's not part of the VMCS. */
13517	break;
13518
13519	case 3:
13520	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13521	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13522	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13523	break;
13524
13525	case 4:
13526	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13527	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13528	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13529	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13530	break;
13531
13532	case 8:
13533	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13534	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13535	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13536	break;
13537
13538	default:
13539	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13540	break;
13541	}
13542
13543	if (rcStrict == VINF_IEM_RAISED_XCPT)
13544	{
13545	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13546	rcStrict = VINF_SUCCESS;
13547	}
13548	return rcStrict;
13549	}
13550
13551
13552	/**
13553	* VM-exit exception handler for \#PF (Page-fault exception).
13554	*
13555	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13556	*/
13557	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13558	{
13559	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13560	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13561	hmR0VmxReadExitQualVmcs(pVmxTransient);
13562
13563	if (!pVM->hm.s.fNestedPaging)
13564	{ /* likely */ }
13565	else
13566	{
13567	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13568	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13569	#endif
13570	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13571	if (!pVmxTransient->fVectoringDoublePF)
13572	{
13573	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13574	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13575	}
13576	else
13577	{
13578	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13579	Assert(!pVmxTransient->fIsNestedGuest);
13580	hmR0VmxSetPendingXcptDF(pVCpu);
13581	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13582	}
13583	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13584	return VINF_SUCCESS;
13585	}
13586
13587	Assert(!pVmxTransient->fIsNestedGuest);
13588
13589	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13590	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13591	if (pVmxTransient->fVectoringPF)
13592	{
13593	Assert(pVCpu->hm.s.Event.fPending);
13594	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13595	}
13596
13597	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13598	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13599	AssertRCReturn(rc, rc);
13600
13601	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
13602	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
13603
13604	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13605	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13606
13607	Log4Func(("#PF: rc=%Rrc\n", rc));
13608	if (rc == VINF_SUCCESS)
13609	{
13610	/*
13611	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13612	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13613	*/
13614	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13615	TRPMResetTrap(pVCpu);
13616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13617	return rc;
13618	}
13619
13620	if (rc == VINF_EM_RAW_GUEST_TRAP)
13621	{
13622	if (!pVmxTransient->fVectoringDoublePF)
13623	{
13624	/* It's a guest page fault and needs to be reflected to the guest. */
13625	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
13626	TRPMResetTrap(pVCpu);
13627	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13628	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13629	uGstErrorCode, pVmxTransient->uExitQual);
13630	}
13631	else
13632	{
13633	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13634	TRPMResetTrap(pVCpu);
13635	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13636	hmR0VmxSetPendingXcptDF(pVCpu);
13637	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13638	}
13639
13640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13641	return VINF_SUCCESS;
13642	}
13643
13644	TRPMResetTrap(pVCpu);
13645	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13646	return rc;
13647	}
13648
13649
13650	/**
13651	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13652	*
13653	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13654	*/
13655	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13656	{
13657	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13659
13660	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13661	AssertRCReturn(rc, rc);
13662
13663	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13664	{
13665	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13666	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13667
13668	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13669	* provides VM-exit instruction length. If this causes problem later,
13670	* disassemble the instruction like it's done on AMD-V. */
13671	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13672	AssertRCReturn(rc2, rc2);
13673	return rc;
13674	}
13675
13676	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
13677	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13678	return VINF_SUCCESS;
13679	}
13680
13681
13682	/**
13683	* VM-exit exception handler for \#BP (Breakpoint exception).
13684	*
13685	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13686	*/
13687	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13688	{
13689	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13690	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13691
13692	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13693	AssertRCReturn(rc, rc);
13694
13695	if (!pVmxTransient->fIsNestedGuest)
13696	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13697	else
13698	rc = VINF_EM_RAW_GUEST_TRAP;
13699
13700	if (rc == VINF_EM_RAW_GUEST_TRAP)
13701	{
13702	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13703	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13704	rc = VINF_SUCCESS;
13705	}
13706
13707	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
13708	return rc;
13709	}
13710
13711
13712	/**
13713	* VM-exit exception handler for \#AC (Alignment-check exception).
13714	*
13715	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13716	*/
13717	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13718	{
13719	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13720	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13721
13722	/* Re-inject it. We'll detect any nesting before getting here. */
13723	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13724	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13725	return VINF_SUCCESS;
13726	}
13727
13728
13729	/**
13730	* VM-exit exception handler for \#DB (Debug exception).
13731	*
13732	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13733	*/
13734	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13735	{
13736	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13738
13739	/*
13740	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13741	*/
13742	hmR0VmxReadExitQualVmcs(pVmxTransient);
13743
13744	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13745	uint64_t const uDR6 = X86_DR6_INIT_VAL
13746	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13747	\| X86_DR6_BD \| X86_DR6_BS));
13748
13749	int rc;
13750	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13751	if (!pVmxTransient->fIsNestedGuest)
13752	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13753	else
13754	rc = VINF_EM_RAW_GUEST_TRAP;
13755	Log6Func(("rc=%Rrc\n", rc));
13756	if (rc == VINF_EM_RAW_GUEST_TRAP)
13757	{
13758	/*
13759	* The exception was for the guest. Update DR6, DR7.GD and
13760	* IA32_DEBUGCTL.LBR before forwarding it.
13761	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
13762	*/
13763	VMMRZCallRing3Disable(pVCpu);
13764	HM_DISABLE_PREEMPT(pVCpu);
13765
13766	pCtx->dr[6] &= ~X86_DR6_B_MASK;
13767	pCtx->dr[6] \|= uDR6;
13768	if (CPUMIsGuestDebugStateActive(pVCpu))
13769	ASMSetDR6(pCtx->dr[6]);
13770
13771	HM_RESTORE_PREEMPT();
13772	VMMRZCallRing3Enable(pVCpu);
13773
13774	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
13775	AssertRCReturn(rc, rc);
13776
13777	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13778	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
13779
13780	/* Paranoia. */
13781	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
13782	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
13783
13784	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
13785	AssertRC(rc);
13786
13787	/*
13788	* Raise #DB in the guest.
13789	*
13790	* It is important to reflect exactly what the VM-exit gave us (preserving the
13791	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13792	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13793	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13794	*
13795	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13796	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13797	*/
13798	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13799	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13800	return VINF_SUCCESS;
13801	}
13802
13803	/*
13804	* Not a guest trap, must be a hypervisor related debug event then.
13805	* Update DR6 in case someone is interested in it.
13806	*/
13807	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13808	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13809	CPUMSetHyperDR6(pVCpu, uDR6);
13810
13811	return rc;
13812	}
13813
13814
13815	/**
13816	* Hacks its way around the lovely mesa driver's backdoor accesses.
13817	*
13818	* @sa hmR0SvmHandleMesaDrvGp.
13819	*/
13820	static int hmR0VmxHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13821	{
13822	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
13823	RT_NOREF(pCtx);
13824
13825	/* For now we'll just skip the instruction. */
13826	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13827	}
13828
13829
13830	/**
13831	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
13832	* backdoor logging w/o checking what it is running inside.
13833	*
13834	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
13835	* backdoor port and magic numbers loaded in registers.
13836	*
13837	* @returns true if it is, false if it isn't.
13838	* @sa hmR0SvmIsMesaDrvGp.
13839	*/
13840	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13841	{
13842	/* 0xed: IN eAX,dx */
13843	uint8_t abInstr[1];
13844	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
13845	return false;
13846
13847	/* Check that it is #GP(0). */
13848	if (pVmxTransient->uExitIntErrorCode != 0)
13849	return false;
13850
13851	/* Check magic and port. */
13852	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
13853	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
13854	if (pCtx->rax != UINT32_C(0x564d5868))
13855	return false;
13856	if (pCtx->dx != UINT32_C(0x5658))
13857	return false;
13858
13859	/* Flat ring-3 CS. */
13860	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
13861	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
13862	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
13863	if (pCtx->cs.Attr.n.u2Dpl != 3)
13864	return false;
13865	if (pCtx->cs.u64Base != 0)
13866	return false;
13867
13868	/* Check opcode. */
13869	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
13870	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
13871	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
13872	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
13873	if (RT_FAILURE(rc))
13874	return false;
13875	if (abInstr[0] != 0xed)
13876	return false;
13877
13878	return true;
13879	}
13880
13881
13882	/**
13883	* VM-exit exception handler for \#GP (General-protection exception).
13884	*
13885	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13886	*/
13887	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13888	{
13889	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13890	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13891
13892	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13893	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13894	if (pVmcsInfo->RealMode.fRealOnV86Active)
13895	{ /* likely */ }
13896	else
13897	{
13898	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13899	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
13900	#endif
13901	/*
13902	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
13903	* executing a nested-guest, reflect #GP to the guest or nested-guest.
13904	*/
13905	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13906	AssertRCReturn(rc, rc);
13907	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13908	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13909
13910	if ( pVmxTransient->fIsNestedGuest
13911	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
13912	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
13913	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13914	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13915	else
13916	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
13917	return rc;
13918	}
13919
13920	Assert(CPUMIsGuestInRealModeEx(pCtx));
13921	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13922	Assert(!pVmxTransient->fIsNestedGuest);
13923
13924	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13925	AssertRCReturn(rc, rc);
13926
13927	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13928	if (rcStrict == VINF_SUCCESS)
13929	{
13930	if (!CPUMIsGuestInRealModeEx(pCtx))
13931	{
13932	/*
13933	* The guest is no longer in real-mode, check if we can continue executing the
13934	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13935	*/
13936	pVmcsInfo->RealMode.fRealOnV86Active = false;
13937	if (HMCanExecuteVmxGuest(pVCpu->pVMR0, pVCpu, pCtx))
13938	{
13939	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
13940	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13941	}
13942	else
13943	{
13944	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13945	rcStrict = VINF_EM_RESCHEDULE;
13946	}
13947	}
13948	else
13949	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13950	}
13951	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13952	{
13953	rcStrict = VINF_SUCCESS;
13954	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13955	}
13956	return VBOXSTRICTRC_VAL(rcStrict);
13957	}
13958
13959
13960	/**
13961	* VM-exit exception handler wrapper for all other exceptions that are not handled
13962	* by a specific handler.
13963	*
13964	* This simply re-injects the exception back into the VM without any special
13965	* processing.
13966	*
13967	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13968	*/
13969	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13970	{
13971	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13972
13973	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13974	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13975	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
13976	("uVector=%#x u32XcptBitmap=%#X32\n",
13977	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
13978	NOREF(pVmcsInfo);
13979	#endif
13980
13981	/*
13982	* Re-inject the exception into the guest. This cannot be a double-fault condition which
13983	* would have been handled while checking exits due to event delivery.
13984	*/
13985	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13986
13987	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13988	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13989	AssertRCReturn(rc, rc);
13990	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13991	#endif
13992
13993	#ifdef VBOX_WITH_STATISTICS
13994	switch (uVector)
13995	{
13996	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
13997	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
13998	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
13999	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14000	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14001	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14002	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14003	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14004	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14005	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14006	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14007	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14008	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14009	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14010	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14011	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14012	default:
14013	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14014	break;
14015	}
14016	#endif
14017
14018	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14019	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
14020	NOREF(uVector);
14021
14022	/* Re-inject the original exception into the guest. */
14023	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14024	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14025	return VINF_SUCCESS;
14026	}
14027
14028
14029	/**
14030	* VM-exit exception handler for all exceptions (except NMIs!).
14031	*
14032	* @remarks This may be called for both guests and nested-guests. Take care to not
14033	* make assumptions and avoid doing anything that is not relevant when
14034	* executing a nested-guest (e.g., Mesa driver hacks).
14035	*/
14036	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14037	{
14038	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
14039
14040	/*
14041	* If this VM-exit occurred while delivering an event through the guest IDT, take
14042	* action based on the return code and additional hints (e.g. for page-faults)
14043	* that will be updated in the VMX transient structure.
14044	*/
14045	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14046	if (rcStrict == VINF_SUCCESS)
14047	{
14048	/*
14049	* If an exception caused a VM-exit due to delivery of an event, the original
14050	* event may have to be re-injected into the guest. We shall reinject it and
14051	* continue guest execution. However, page-fault is a complicated case and
14052	* needs additional processing done in hmR0VmxExitXcptPF().
14053	*/
14054	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14055	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14056	if ( !pVCpu->hm.s.Event.fPending
14057	\|\| uVector == X86_XCPT_PF)
14058	{
14059	switch (uVector)
14060	{
14061	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
14062	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
14063	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
14064	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
14065	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
14066	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
14067	default:
14068	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
14069	}
14070	}
14071	/* else: inject pending event before resuming guest execution. */
14072	}
14073	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
14074	{
14075	Assert(pVCpu->hm.s.Event.fPending);
14076	rcStrict = VINF_SUCCESS;
14077	}
14078
14079	return rcStrict;
14080	}
14081	/** @} */
14082
14083
14084	/** @name VM-exit handlers.
14085	* @{
14086	*/
14087	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14088	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14089	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14090
14091	/**
14092	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14093	*/
14094	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14095	{
14096	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14097	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14098	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14099	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14100	return VINF_SUCCESS;
14101	return VINF_EM_RAW_INTERRUPT;
14102	}
14103
14104
14105	/**
14106	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14107	* VM-exit.
14108	*/
14109	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14110	{
14111	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14112	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14113
14114	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14115
14116	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14117	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14118	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14119
14120	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14121	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14122	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14123	NOREF(pVmcsInfo);
14124
14125	VBOXSTRICTRC rcStrict;
14126	switch (uExitIntType)
14127	{
14128	/*
14129	* Host physical NMIs:
14130	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14131	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14132	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14133	*
14134	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14135	* See Intel spec. 27.5.5 "Updating Non-Register State".
14136	*/
14137	case VMX_EXIT_INT_INFO_TYPE_NMI:
14138	{
14139	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14140	break;
14141	}
14142
14143	/*
14144	* Privileged software exceptions (#DB from ICEBP),
14145	* Software exceptions (#BP and #OF),
14146	* Hardware exceptions:
14147	* Process the required exceptions and resume guest execution if possible.
14148	*/
14149	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14150	Assert(uVector == X86_XCPT_DB);
14151	RT_FALL_THRU();
14152	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14153	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14154	RT_FALL_THRU();
14155	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14156	{
14157	NOREF(uVector);
14158	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14159	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14160	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14161	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14162
14163	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14164	break;
14165	}
14166
14167	default:
14168	{
14169	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14170	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14171	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14172	break;
14173	}
14174	}
14175
14176	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14177	return rcStrict;
14178	}
14179
14180
14181	/**
14182	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14183	*/
14184	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14185	{
14186	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14187
14188	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14189	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14190	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14191
14192	/* Evaluate and deliver pending events and resume guest execution. */
14193	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14194	return VINF_SUCCESS;
14195	}
14196
14197
14198	/**
14199	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14200	*/
14201	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14202	{
14203	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14204
14205	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14206	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14207	{
14208	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14209	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14210	}
14211
14212	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14213
14214	/*
14215	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14216	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14217	*/
14218	uint32_t fIntrState;
14219	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14220	AssertRC(rc);
14221	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14222	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14223	{
14224	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14225	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14226
14227	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14228	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14229	AssertRC(rc);
14230	}
14231
14232	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14233	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14234
14235	/* Evaluate and deliver pending events and resume guest execution. */
14236	return VINF_SUCCESS;
14237	}
14238
14239
14240	/**
14241	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14242	*/
14243	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14244	{
14245	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14246	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14247	}
14248
14249
14250	/**
14251	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14252	*/
14253	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14254	{
14255	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14256	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14257	}
14258
14259
14260	/**
14261	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14262	*/
14263	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14264	{
14265	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14266
14267	/*
14268	* Get the state we need and update the exit history entry.
14269	*/
14270	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14271	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14272
14273	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14274	AssertRCReturn(rc, rc);
14275
14276	VBOXSTRICTRC rcStrict;
14277	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14278	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14279	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14280	if (!pExitRec)
14281	{
14282	/*
14283	* Regular CPUID instruction execution.
14284	*/
14285	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
14286	if (rcStrict == VINF_SUCCESS)
14287	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14288	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14289	{
14290	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14291	rcStrict = VINF_SUCCESS;
14292	}
14293	}
14294	else
14295	{
14296	/*
14297	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14298	*/
14299	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14300	AssertRCReturn(rc2, rc2);
14301
14302	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14303	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14304
14305	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14306	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14307
14308	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14309	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14310	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14311	}
14312	return rcStrict;
14313	}
14314
14315
14316	/**
14317	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14318	*/
14319	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14320	{
14321	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14322
14323	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14324	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14325	AssertRCReturn(rc, rc);
14326
14327	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14328	return VINF_EM_RAW_EMULATE_INSTR;
14329
14330	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14331	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14332	}
14333
14334
14335	/**
14336	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14337	*/
14338	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14339	{
14340	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14341
14342	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14343	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14344	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14345	AssertRCReturn(rc, rc);
14346
14347	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
14348	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14349	{
14350	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14351	we must reset offsetting on VM-entry. See @bugref{6634}. */
14352	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14353	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14354	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14355	}
14356	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14357	{
14358	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14359	rcStrict = VINF_SUCCESS;
14360	}
14361	return rcStrict;
14362	}
14363
14364
14365	/**
14366	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14367	*/
14368	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14369	{
14370	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14371
14372	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14373	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14374	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14375	AssertRCReturn(rc, rc);
14376
14377	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
14378	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14379	{
14380	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14381	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14382	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14383	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14384	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14385	}
14386	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14387	{
14388	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14389	rcStrict = VINF_SUCCESS;
14390	}
14391	return rcStrict;
14392	}
14393
14394
14395	/**
14396	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14397	*/
14398	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14399	{
14400	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14401
14402	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14403	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14404	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14405	AssertRCReturn(rc, rc);
14406
14407	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14408	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14409	if (RT_LIKELY(rc == VINF_SUCCESS))
14410	{
14411	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14412	Assert(pVmxTransient->cbExitInstr == 2);
14413	}
14414	else
14415	{
14416	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14417	rc = VERR_EM_INTERPRETER;
14418	}
14419	return rc;
14420	}
14421
14422
14423	/**
14424	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14425	*/
14426	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14427	{
14428	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14429
14430	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14431	if (EMAreHypercallInstructionsEnabled(pVCpu))
14432	{
14433	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14434	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14435	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14436	AssertRCReturn(rc, rc);
14437
14438	/* Perform the hypercall. */
14439	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14440	if (rcStrict == VINF_SUCCESS)
14441	{
14442	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14443	AssertRCReturn(rc, rc);
14444	}
14445	else
14446	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14447	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14448	\|\| RT_FAILURE(rcStrict));
14449
14450	/* If the hypercall changes anything other than guest's general-purpose registers,
14451	we would need to reload the guest changed bits here before VM-entry. */
14452	}
14453	else
14454	Log4Func(("Hypercalls not enabled\n"));
14455
14456	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14457	if (RT_FAILURE(rcStrict))
14458	{
14459	hmR0VmxSetPendingXcptUD(pVCpu);
14460	rcStrict = VINF_SUCCESS;
14461	}
14462
14463	return rcStrict;
14464	}
14465
14466
14467	/**
14468	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14469	*/
14470	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14471	{
14472	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14473	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14474
14475	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14476	hmR0VmxReadExitQualVmcs(pVmxTransient);
14477	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14478	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14479	AssertRCReturn(rc, rc);
14480
14481	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
14482
14483	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14484	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14485	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14486	{
14487	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14488	rcStrict = VINF_SUCCESS;
14489	}
14490	else
14491	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14492	VBOXSTRICTRC_VAL(rcStrict)));
14493	return rcStrict;
14494	}
14495
14496
14497	/**
14498	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14499	*/
14500	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14501	{
14502	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14503
14504	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14505	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14506	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14507	AssertRCReturn(rc, rc);
14508
14509	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
14510	if (rcStrict == VINF_SUCCESS)
14511	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14512	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14513	{
14514	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14515	rcStrict = VINF_SUCCESS;
14516	}
14517
14518	return rcStrict;
14519	}
14520
14521
14522	/**
14523	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14524	*/
14525	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14526	{
14527	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14528
14529	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14530	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14531	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14532	AssertRCReturn(rc, rc);
14533
14534	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
14535	if (RT_SUCCESS(rcStrict))
14536	{
14537	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14538	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14539	rcStrict = VINF_SUCCESS;
14540	}
14541
14542	return rcStrict;
14543	}
14544
14545
14546	/**
14547	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14548	* VM-exit.
14549	*/
14550	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14551	{
14552	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14553	return VINF_EM_RESET;
14554	}
14555
14556
14557	/**
14558	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14559	*/
14560	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14561	{
14562	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14563
14564	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14565	AssertRCReturn(rc, rc);
14566
14567	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14568	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14569	rc = VINF_SUCCESS;
14570	else
14571	rc = VINF_EM_HALT;
14572
14573	if (rc != VINF_SUCCESS)
14574	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14575	return rc;
14576	}
14577
14578
14579	/**
14580	* VM-exit handler for instructions that result in a \#UD exception delivered to
14581	* the guest.
14582	*/
14583	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14584	{
14585	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14586	hmR0VmxSetPendingXcptUD(pVCpu);
14587	return VINF_SUCCESS;
14588	}
14589
14590
14591	/**
14592	* VM-exit handler for expiry of the VMX-preemption timer.
14593	*/
14594	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14595	{
14596	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14597
14598	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14599	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14600
14601	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14602	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14603	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14604	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14605	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14606	}
14607
14608
14609	/**
14610	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14611	*/
14612	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14613	{
14614	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14615
14616	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14617	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14618	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14619	AssertRCReturn(rc, rc);
14620
14621	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
14622	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14623	: HM_CHANGED_RAISED_XCPT_MASK);
14624
14625	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14626	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14627
14628	return rcStrict;
14629	}
14630
14631
14632	/**
14633	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14634	*/
14635	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14636	{
14637	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14638
14639	#if 1
14640	/** @todo Use VM-exit instruction information. */
14641	return VERR_EM_INTERPRETER;
14642	#else
14643	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14644	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14645	hmR0VmxReadExitQualVmcs(pVmxTransient);
14646	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
14647	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14648	AssertRCReturn(rc, rc);
14649
14650	/* Paranoia. Ensure this has a memory operand. */
14651	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
14652
14653	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
14654	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14655	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
14656	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
14657
14658	RTGCPTR GCPtrDesc;
14659	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
14660
14661	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
14662	GCPtrDesc, uType);
14663	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14664	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14665	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14666	{
14667	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14668	rcStrict = VINF_SUCCESS;
14669	}
14670	return rcStrict;
14671	#endif
14672	}
14673
14674
14675	/**
14676	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14677	* VM-exit.
14678	*/
14679	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14680	{
14681	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14682	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14683	AssertRCReturn(rc, rc);
14684
14685	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14686	if (RT_FAILURE(rc))
14687	return rc;
14688
14689	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14690	NOREF(uInvalidReason);
14691
14692	#ifdef VBOX_STRICT
14693	uint32_t fIntrState;
14694	uint64_t u64Val;
14695	hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14696	hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14697	hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14698
14699	Log4(("uInvalidReason %u\n", uInvalidReason));
14700	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14701	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14702	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14703
14704	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
14705	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14706	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
14707	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
14708	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
14709	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
14710	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
14711	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14712	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
14713	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
14714	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
14715	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14716	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14717	{
14718	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14719	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14720	}
14721	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
14722	#endif
14723
14724	return VERR_VMX_INVALID_GUEST_STATE;
14725	}
14726
14727	/**
14728	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14729	*/
14730	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14731	{
14732	/*
14733	* Cumulative notes of all recognized but unexpected VM-exits.
14734	*
14735	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
14736	* nested-paging is used.
14737	*
14738	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14739	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14740	* this function (and thereby stop VM execution) for handling such instructions.
14741	*
14742	*
14743	* VMX_EXIT_INIT_SIGNAL:
14744	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14745	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14746	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14747	*
14748	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14749	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14750	* See Intel spec. "23.8 Restrictions on VMX operation".
14751	*
14752	* VMX_EXIT_SIPI:
14753	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14754	* activity state is used. We don't make use of it as our guests don't have direct
14755	* access to the host local APIC.
14756	*
14757	* See Intel spec. 25.3 "Other Causes of VM-exits".
14758	*
14759	* VMX_EXIT_IO_SMI:
14760	* VMX_EXIT_SMI:
14761	* This can only happen if we support dual-monitor treatment of SMI, which can be
14762	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14763	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14764	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14765	*
14766	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14767	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14768	*
14769	* VMX_EXIT_ERR_MSR_LOAD:
14770	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14771	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14772	* execution.
14773	*
14774	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14775	*
14776	* VMX_EXIT_ERR_MACHINE_CHECK:
14777	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14778	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14779	* #MC exception abort class exception is raised. We thus cannot assume a
14780	* reasonable chance of continuing any sort of execution and we bail.
14781	*
14782	* See Intel spec. 15.1 "Machine-check Architecture".
14783	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14784	*
14785	* VMX_EXIT_PML_FULL:
14786	* VMX_EXIT_VIRTUALIZED_EOI:
14787	* VMX_EXIT_APIC_WRITE:
14788	* We do not currently support any of these features and thus they are all unexpected
14789	* VM-exits.
14790	*
14791	* VMX_EXIT_GDTR_IDTR_ACCESS:
14792	* VMX_EXIT_LDTR_TR_ACCESS:
14793	* VMX_EXIT_RDRAND:
14794	* VMX_EXIT_RSM:
14795	* VMX_EXIT_VMFUNC:
14796	* VMX_EXIT_ENCLS:
14797	* VMX_EXIT_RDSEED:
14798	* VMX_EXIT_XSAVES:
14799	* VMX_EXIT_XRSTORS:
14800	* VMX_EXIT_UMWAIT:
14801	* VMX_EXIT_TPAUSE:
14802	* These VM-exits are -not- caused unconditionally by execution of the corresponding
14803	* instruction. Any VM-exit for these instructions indicate a hardware problem,
14804	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14805	*
14806	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14807	*/
14808	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14809	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14810	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14811	}
14812
14813
14814	/**
14815	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14816	*/
14817	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14818	{
14819	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14820
14821	/** @todo Optimize this: We currently drag in the whole MSR state
14822	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14823	* MSRs required. That would require changes to IEM and possibly CPUM too.
14824	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14825	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14826	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14827	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14828	switch (idMsr)
14829	{
14830	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14831	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14832	}
14833
14834	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14835	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14836	AssertRCReturn(rc, rc);
14837
14838	Log4Func(("ecx=%#RX32\n", idMsr));
14839
14840	#ifdef VBOX_STRICT
14841	Assert(!pVmxTransient->fIsNestedGuest);
14842	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
14843	{
14844	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14845	&& idMsr != MSR_K6_EFER)
14846	{
14847	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14848	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14849	}
14850	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14851	{
14852	Assert(pVmcsInfo->pvMsrBitmap);
14853	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14854	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14855	{
14856	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14857	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14858	}
14859	}
14860	}
14861	#endif
14862
14863	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
14864	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14865	if (rcStrict == VINF_SUCCESS)
14866	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14867	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14868	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14869	{
14870	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14871	rcStrict = VINF_SUCCESS;
14872	}
14873	else
14874	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14875
14876	return rcStrict;
14877	}
14878
14879
14880	/**
14881	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14882	*/
14883	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14884	{
14885	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14886
14887	/** @todo Optimize this: We currently drag in the whole MSR state
14888	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14889	* MSRs required. That would require changes to IEM and possibly CPUM too.
14890	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14891	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14892	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14893
14894	/*
14895	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14896	* Although we don't need to fetch the base as it will be overwritten shortly, while
14897	* loading guest-state we would also load the entire segment register including limit
14898	* and attributes and thus we need to load them here.
14899	*/
14900	switch (idMsr)
14901	{
14902	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14903	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14904	}
14905
14906	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14907	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14908	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14909	AssertRCReturn(rc, rc);
14910
14911	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14912
14913	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
14914	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14915
14916	if (rcStrict == VINF_SUCCESS)
14917	{
14918	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14919
14920	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14921	if ( idMsr == MSR_IA32_APICBASE
14922	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14923	&& idMsr <= MSR_IA32_X2APIC_END))
14924	{
14925	/*
14926	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14927	* When full APIC register virtualization is implemented we'll have to make
14928	* sure APIC state is saved from the VMCS before IEM changes it.
14929	*/
14930	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14931	}
14932	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14933	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14934	else if (idMsr == MSR_K6_EFER)
14935	{
14936	/*
14937	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14938	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14939	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14940	*/
14941	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14942	}
14943
14944	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
14945	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14946	{
14947	switch (idMsr)
14948	{
14949	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14950	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14951	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14952	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14953	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14954	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14955	default:
14956	{
14957	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14958	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14959	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14960	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14961	break;
14962	}
14963	}
14964	}
14965	#ifdef VBOX_STRICT
14966	else
14967	{
14968	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14969	switch (idMsr)
14970	{
14971	case MSR_IA32_SYSENTER_CS:
14972	case MSR_IA32_SYSENTER_EIP:
14973	case MSR_IA32_SYSENTER_ESP:
14974	case MSR_K8_FS_BASE:
14975	case MSR_K8_GS_BASE:
14976	{
14977	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14978	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14979	}
14980
14981	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14982	default:
14983	{
14984	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14985	{
14986	/* EFER MSR writes are always intercepted. */
14987	if (idMsr != MSR_K6_EFER)
14988	{
14989	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14990	idMsr));
14991	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14992	}
14993	}
14994
14995	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14996	{
14997	Assert(pVmcsInfo->pvMsrBitmap);
14998	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14999	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15000	{
15001	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15002	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15003	}
15004	}
15005	break;
15006	}
15007	}
15008	}
15009	#endif /* VBOX_STRICT */
15010	}
15011	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15012	{
15013	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15014	rcStrict = VINF_SUCCESS;
15015	}
15016	else
15017	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15018
15019	return rcStrict;
15020	}
15021
15022
15023	/**
15024	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15025	*/
15026	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15027	{
15028	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15029
15030	/** @todo The guest has likely hit a contended spinlock. We might want to
15031	* poke a schedule different guest VCPU. */
15032	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15033	if (RT_SUCCESS(rc))
15034	return VINF_EM_RAW_INTERRUPT;
15035
15036	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15037	return rc;
15038	}
15039
15040
15041	/**
15042	* VM-exit handler for when the TPR value is lowered below the specified
15043	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15044	*/
15045	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15046	{
15047	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15048	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15049
15050	/*
15051	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15052	* We'll re-evaluate pending interrupts and inject them before the next VM
15053	* entry so we can just continue execution here.
15054	*/
15055	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15056	return VINF_SUCCESS;
15057	}
15058
15059
15060	/**
15061	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15062	* VM-exit.
15063	*
15064	* @retval VINF_SUCCESS when guest execution can continue.
15065	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15066	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15067	* incompatible guest state for VMX execution (real-on-v86 case).
15068	*/
15069	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15070	{
15071	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15072	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15073
15074	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15075	hmR0VmxReadExitQualVmcs(pVmxTransient);
15076	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15077
15078	VBOXSTRICTRC rcStrict;
15079	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15080	uint64_t const uExitQual = pVmxTransient->uExitQual;
15081	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15082	switch (uAccessType)
15083	{
15084	/*
15085	* MOV to CRx.
15086	*/
15087	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15088	{
15089	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15090	AssertRCReturn(rc, rc);
15091
15092	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15093	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15094	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15095	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15096
15097	/*
15098	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15099	* - When nested paging isn't used.
15100	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15101	* - We are executing in the VM debug loop.
15102	*/
15103	Assert( iCrReg != 3
15104	\|\| !pVM->hm.s.fNestedPaging
15105	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15106	\|\| pVCpu->hm.s.fUsingDebugLoop);
15107
15108	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15109	Assert( iCrReg != 8
15110	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15111
15112	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15113	AssertMsg( rcStrict == VINF_SUCCESS
15114	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15115
15116	/*
15117	* This is a kludge for handling switches back to real mode when we try to use
15118	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15119	* deal with special selector values, so we have to return to ring-3 and run
15120	* there till the selector values are V86 mode compatible.
15121	*
15122	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15123	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15124	* this function.
15125	*/
15126	if ( iCrReg == 0
15127	&& rcStrict == VINF_SUCCESS
15128	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15129	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15130	&& (uOldCr0 & X86_CR0_PE)
15131	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15132	{
15133	/** @todo Check selectors rather than returning all the time. */
15134	Assert(!pVmxTransient->fIsNestedGuest);
15135	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15136	rcStrict = VINF_EM_RESCHEDULE_REM;
15137	}
15138	break;
15139	}
15140
15141	/*
15142	* MOV from CRx.
15143	*/
15144	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15145	{
15146	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15147	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15148
15149	/*
15150	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15151	* - When nested paging isn't used.
15152	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15153	* - We are executing in the VM debug loop.
15154	*/
15155	Assert( iCrReg != 3
15156	\|\| !pVM->hm.s.fNestedPaging
15157	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15158	\|\| pVCpu->hm.s.fUsingDebugLoop);
15159
15160	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15161	Assert( iCrReg != 8
15162	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15163
15164	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15165	break;
15166	}
15167
15168	/*
15169	* CLTS (Clear Task-Switch Flag in CR0).
15170	*/
15171	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15172	{
15173	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
15174	break;
15175	}
15176
15177	/*
15178	* LMSW (Load Machine-Status Word into CR0).
15179	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15180	*/
15181	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15182	{
15183	RTGCPTR GCPtrEffDst;
15184	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
15185	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15186	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15187	if (fMemOperand)
15188	{
15189	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
15190	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15191	}
15192	else
15193	GCPtrEffDst = NIL_RTGCPTR;
15194	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15195	break;
15196	}
15197
15198	default:
15199	{
15200	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15201	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15202	}
15203	}
15204
15205	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15206	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15207	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15208
15209	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15210	NOREF(pVM);
15211	return rcStrict;
15212	}
15213
15214
15215	/**
15216	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15217	* VM-exit.
15218	*/
15219	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15220	{
15221	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15222	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15223
15224	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15225	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15226	hmR0VmxReadExitQualVmcs(pVmxTransient);
15227	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15228	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15229	\| CPUMCTX_EXTRN_EFER);
15230	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15231	AssertRCReturn(rc, rc);
15232
15233	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15234	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15235	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15236	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15237	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15238	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15239	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15240	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15241
15242	/*
15243	* Update exit history to see if this exit can be optimized.
15244	*/
15245	VBOXSTRICTRC rcStrict;
15246	PCEMEXITREC pExitRec = NULL;
15247	if ( !fGstStepping
15248	&& !fDbgStepping)
15249	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15250	!fIOString
15251	? !fIOWrite
15252	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15253	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15254	: !fIOWrite
15255	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15256	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15257	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15258	if (!pExitRec)
15259	{
15260	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15261	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15262
15263	uint32_t const cbValue = s_aIOSizes[uIOSize];
15264	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
15265	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15266	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15267	if (fIOString)
15268	{
15269	/*
15270	* INS/OUTS - I/O String instruction.
15271	*
15272	* Use instruction-information if available, otherwise fall back on
15273	* interpreting the instruction.
15274	*/
15275	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15276	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15277	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15278	if (fInsOutsInfo)
15279	{
15280	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15281	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15282	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15283	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15284	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15285	if (fIOWrite)
15286	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15287	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15288	else
15289	{
15290	/*
15291	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15292	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15293	* See Intel Instruction spec. for "INS".
15294	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15295	*/
15296	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15297	}
15298	}
15299	else
15300	rcStrict = IEMExecOne(pVCpu);
15301
15302	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15303	fUpdateRipAlready = true;
15304	}
15305	else
15306	{
15307	/*
15308	* IN/OUT - I/O instruction.
15309	*/
15310	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15311	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15312	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15313	if (fIOWrite)
15314	{
15315	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15317	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15318	&& !pCtx->eflags.Bits.u1TF)
15319	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15320	}
15321	else
15322	{
15323	uint32_t u32Result = 0;
15324	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15325	if (IOM_SUCCESS(rcStrict))
15326	{
15327	/* Save result of I/O IN instr. in AL/AX/EAX. */
15328	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15329	}
15330	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15331	&& !pCtx->eflags.Bits.u1TF)
15332	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15334	}
15335	}
15336
15337	if (IOM_SUCCESS(rcStrict))
15338	{
15339	if (!fUpdateRipAlready)
15340	{
15341	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15342	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15343	}
15344
15345	/*
15346	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15347	* while booting Fedora 17 64-bit guest.
15348	*
15349	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15350	*/
15351	if (fIOString)
15352	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15353
15354	/*
15355	* If any I/O breakpoints are armed, we need to check if one triggered
15356	* and take appropriate action.
15357	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15358	*/
15359	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15360	AssertRCReturn(rc, rc);
15361
15362	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15363	* execution engines about whether hyper BPs and such are pending. */
15364	uint32_t const uDr7 = pCtx->dr[7];
15365	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15366	&& X86_DR7_ANY_RW_IO(uDr7)
15367	&& (pCtx->cr4 & X86_CR4_DE))
15368	\|\| DBGFBpIsHwIoArmed(pVM)))
15369	{
15370	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15371
15372	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15373	VMMRZCallRing3Disable(pVCpu);
15374	HM_DISABLE_PREEMPT(pVCpu);
15375
15376	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15377
15378	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15379	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15380	{
15381	/* Raise #DB. */
15382	if (fIsGuestDbgActive)
15383	ASMSetDR6(pCtx->dr[6]);
15384	if (pCtx->dr[7] != uDr7)
15385	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15386
15387	hmR0VmxSetPendingXcptDB(pVCpu);
15388	}
15389	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15390	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15391	else if ( rcStrict2 != VINF_SUCCESS
15392	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15393	rcStrict = rcStrict2;
15394	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15395
15396	HM_RESTORE_PREEMPT();
15397	VMMRZCallRing3Enable(pVCpu);
15398	}
15399	}
15400
15401	#ifdef VBOX_STRICT
15402	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15403	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15404	Assert(!fIOWrite);
15405	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15406	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15407	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15408	Assert(fIOWrite);
15409	else
15410	{
15411	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15412	* statuses, that the VMM device and some others may return. See
15413	* IOM_SUCCESS() for guidance. */
15414	AssertMsg( RT_FAILURE(rcStrict)
15415	\|\| rcStrict == VINF_SUCCESS
15416	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15417	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15418	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15419	\|\| rcStrict == VINF_EM_RAW_TO_R3
15420	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15421	# endif
15422	}
15423	#endif
15424	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15425	}
15426	else
15427	{
15428	/*
15429	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15430	*/
15431	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15432	AssertRCReturn(rc2, rc2);
15433	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15434	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15435	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15436	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15437	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15438	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15439
15440	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15441	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15442
15443	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15444	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15445	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15446	}
15447	return rcStrict;
15448	}
15449
15450
15451	/**
15452	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15453	* VM-exit.
15454	*/
15455	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15456	{
15457	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15458
15459	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15460	hmR0VmxReadExitQualVmcs(pVmxTransient);
15461	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15462	{
15463	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15464	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15465	{
15466	uint32_t uErrCode;
15467	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15468	{
15469	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15470	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15471	}
15472	else
15473	uErrCode = 0;
15474
15475	RTGCUINTPTR GCPtrFaultAddress;
15476	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15477	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15478	else
15479	GCPtrFaultAddress = 0;
15480
15481	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15482
15483	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15484	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
15485
15486	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15487	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15488	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15489	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15490	}
15491	}
15492
15493	/* Fall back to the interpreter to emulate the task-switch. */
15494	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15495	return VERR_EM_INTERPRETER;
15496	}
15497
15498
15499	/**
15500	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15501	*/
15502	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15503	{
15504	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15505
15506	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15507	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15508	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15509	AssertRC(rc);
15510	return VINF_EM_DBG_STEPPED;
15511	}
15512
15513
15514	/**
15515	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15516	*/
15517	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15518	{
15519	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15520	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15521
15522	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15523	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15524	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15525	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15526	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15527
15528	/*
15529	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15530	*/
15531	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15532	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15533	{
15534	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15535	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15536	{
15537	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15538	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15539	}
15540	}
15541	else
15542	{
15543	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15544	return rcStrict;
15545	}
15546
15547	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15548	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15549	hmR0VmxReadExitQualVmcs(pVmxTransient);
15550	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15551	AssertRCReturn(rc, rc);
15552
15553	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15554	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15555	switch (uAccessType)
15556	{
15557	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15558	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15559	{
15560	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15561	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15562	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15563
15564	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15565	GCPhys &= PAGE_BASE_GC_MASK;
15566	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15567	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15568	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15569
15570	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15571	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15572	rcStrict = IOMMMIOPhysHandler(pVM, pVCpu,
15573	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15574	CPUMCTX2CORE(pCtx), GCPhys);
15575	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15576	if ( rcStrict == VINF_SUCCESS
15577	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15578	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15579	{
15580	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15581	\| HM_CHANGED_GUEST_APIC_TPR);
15582	rcStrict = VINF_SUCCESS;
15583	}
15584	break;
15585	}
15586
15587	default:
15588	{
15589	Log4Func(("uAccessType=%#x\n", uAccessType));
15590	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15591	break;
15592	}
15593	}
15594
15595	if (rcStrict != VINF_SUCCESS)
15596	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15597	return rcStrict;
15598	}
15599
15600
15601	/**
15602	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15603	* VM-exit.
15604	*/
15605	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15606	{
15607	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15608	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15609
15610	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15611	if (!pVmxTransient->fIsNestedGuest)
15612	{
15613	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15614	if (pVmxTransient->fWasGuestDebugStateActive)
15615	{
15616	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15617	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15618	}
15619
15620	if ( !pVCpu->hm.s.fSingleInstruction
15621	&& !pVmxTransient->fWasHyperDebugStateActive)
15622	{
15623	Assert(!DBGFIsStepping(pVCpu));
15624	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15625
15626	/* Don't intercept MOV DRx any more. */
15627	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15628	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15629	AssertRC(rc);
15630
15631	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15632	VMMRZCallRing3Disable(pVCpu);
15633	HM_DISABLE_PREEMPT(pVCpu);
15634
15635	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15636	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15637	Assert(CPUMIsGuestDebugStateActive(pVCpu));
15638
15639	HM_RESTORE_PREEMPT();
15640	VMMRZCallRing3Enable(pVCpu);
15641
15642	#ifdef VBOX_WITH_STATISTICS
15643	hmR0VmxReadExitQualVmcs(pVmxTransient);
15644	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15645	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15646	else
15647	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15648	#endif
15649	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15650	return VINF_SUCCESS;
15651	}
15652	}
15653
15654	/*
15655	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15656	* The EFER MSR is always up-to-date.
15657	* Update the segment registers and DR7 from the CPU.
15658	*/
15659	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15660	hmR0VmxReadExitQualVmcs(pVmxTransient);
15661	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15662	AssertRCReturn(rc, rc);
15663	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15664
15665	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15666	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15667	{
15668	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15669	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15670	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15671	if (RT_SUCCESS(rc))
15672	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15674	}
15675	else
15676	{
15677	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15678	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15679	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15680	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15681	}
15682
15683	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15684	if (RT_SUCCESS(rc))
15685	{
15686	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15687	AssertRCReturn(rc2, rc2);
15688	return VINF_SUCCESS;
15689	}
15690	return rc;
15691	}
15692
15693
15694	/**
15695	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15696	* Conditional VM-exit.
15697	*/
15698	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15699	{
15700	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15701	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15702
15703	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15704	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15705	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15706	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15707	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15708
15709	/*
15710	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15711	*/
15712	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15713	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15714	{
15715	/*
15716	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15717	* instruction emulation to inject the original event. Otherwise, injecting the original event
15718	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
15719	*/
15720	if (!pVCpu->hm.s.Event.fPending)
15721	{ /* likely */ }
15722	else
15723	{
15724	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15725	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15726	/** @todo NSTVMX: Think about how this should be handled. */
15727	if (pVmxTransient->fIsNestedGuest)
15728	return VERR_VMX_IPE_3;
15729	#endif
15730	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15731	}
15732	}
15733	else
15734	{
15735	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15736	return rcStrict;
15737	}
15738
15739	/*
15740	* Get sufficient state and update the exit history entry.
15741	*/
15742	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15743	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15744	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15745	AssertRCReturn(rc, rc);
15746
15747	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15748	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15749	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15750	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15751	if (!pExitRec)
15752	{
15753	/*
15754	* If we succeed, resume guest execution.
15755	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15756	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15757	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15758	* weird case. See @bugref{6043}.
15759	*/
15760	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15761	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15762	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15763	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15764	if ( rcStrict == VINF_SUCCESS
15765	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15766	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15767	{
15768	/* Successfully handled MMIO operation. */
15769	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15770	\| HM_CHANGED_GUEST_APIC_TPR);
15771	rcStrict = VINF_SUCCESS;
15772	}
15773	}
15774	else
15775	{
15776	/*
15777	* Frequent exit or something needing probing. Call EMHistoryExec.
15778	*/
15779	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15780	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15781
15782	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15783	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15784
15785	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15786	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15787	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15788	}
15789	return rcStrict;
15790	}
15791
15792
15793	/**
15794	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15795	* VM-exit.
15796	*/
15797	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15798	{
15799	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15800	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15801
15802	hmR0VmxReadExitQualVmcs(pVmxTransient);
15803	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15804	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15805	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15806	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15807	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15808
15809	/*
15810	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15811	*/
15812	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15813	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15814	{
15815	/*
15816	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
15817	* we shall resolve the nested #PF and re-inject the original event.
15818	*/
15819	if (pVCpu->hm.s.Event.fPending)
15820	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
15821	}
15822	else
15823	{
15824	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15825	return rcStrict;
15826	}
15827
15828	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15829	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15830	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15831	AssertRCReturn(rc, rc);
15832
15833	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15834	uint64_t const uExitQual = pVmxTransient->uExitQual;
15835	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
15836
15837	RTGCUINT uErrorCode = 0;
15838	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15839	uErrorCode \|= X86_TRAP_PF_ID;
15840	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15841	uErrorCode \|= X86_TRAP_PF_RW;
15842	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15843	uErrorCode \|= X86_TRAP_PF_P;
15844
15845	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15846	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15847	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
15848
15849	/*
15850	* Handle the pagefault trap for the nested shadow table.
15851	*/
15852	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15853	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15854	TRPMResetTrap(pVCpu);
15855
15856	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15857	if ( rcStrict == VINF_SUCCESS
15858	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15859	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15860	{
15861	/* Successfully synced our nested page tables. */
15862	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15863	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15864	return VINF_SUCCESS;
15865	}
15866
15867	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15868	return rcStrict;
15869	}
15870
15871
15872	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15873	/**
15874	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15875	*/
15876	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15877	{
15878	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15879
15880	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15881	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15882	hmR0VmxReadExitQualVmcs(pVmxTransient);
15883	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15884	\| CPUMCTX_EXTRN_HWVIRT
15885	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15886	AssertRCReturn(rc, rc);
15887
15888	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15889
15890	VMXVEXITINFO ExitInfo;
15891	RT_ZERO(ExitInfo);
15892	ExitInfo.uReason = pVmxTransient->uExitReason;
15893	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15894	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15895	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15896	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15897
15898	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15899	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15900	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15901	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15902	{
15903	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15904	rcStrict = VINF_SUCCESS;
15905	}
15906	return rcStrict;
15907	}
15908
15909
15910	/**
15911	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15912	*/
15913	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15914	{
15915	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15916
15917	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15918	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15919	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15920	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15921	AssertRCReturn(rc, rc);
15922
15923	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15924
15925	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
15926	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
15927	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
15928	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15929	{
15930	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15931	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
15932	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15933	}
15934	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15935	return rcStrict;
15936	}
15937
15938
15939	/**
15940	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15941	*/
15942	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15943	{
15944	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15945
15946	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15947	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15948	hmR0VmxReadExitQualVmcs(pVmxTransient);
15949	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15950	\| CPUMCTX_EXTRN_HWVIRT
15951	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15952	AssertRCReturn(rc, rc);
15953
15954	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15955
15956	VMXVEXITINFO ExitInfo;
15957	RT_ZERO(ExitInfo);
15958	ExitInfo.uReason = pVmxTransient->uExitReason;
15959	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15960	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15961	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15962	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15963
15964	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15965	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15967	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15968	{
15969	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15970	rcStrict = VINF_SUCCESS;
15971	}
15972	return rcStrict;
15973	}
15974
15975
15976	/**
15977	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15978	*/
15979	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15980	{
15981	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15982
15983	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15984	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15985	hmR0VmxReadExitQualVmcs(pVmxTransient);
15986	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15987	\| CPUMCTX_EXTRN_HWVIRT
15988	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15989	AssertRCReturn(rc, rc);
15990
15991	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15992
15993	VMXVEXITINFO ExitInfo;
15994	RT_ZERO(ExitInfo);
15995	ExitInfo.uReason = pVmxTransient->uExitReason;
15996	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15997	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15998	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15999	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16000
16001	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16002	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16003	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16004	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16005	{
16006	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16007	rcStrict = VINF_SUCCESS;
16008	}
16009	return rcStrict;
16010	}
16011
16012
16013	/**
16014	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16015	*/
16016	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16017	{
16018	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16019
16020	/*
16021	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16022	* thus might not need to import the shadow VMCS state, it's safer just in case
16023	* code elsewhere dares look at unsynced VMCS fields.
16024	*/
16025	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16026	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16027	hmR0VmxReadExitQualVmcs(pVmxTransient);
16028	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16029	\| CPUMCTX_EXTRN_HWVIRT
16030	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16031	AssertRCReturn(rc, rc);
16032
16033	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16034
16035	VMXVEXITINFO ExitInfo;
16036	RT_ZERO(ExitInfo);
16037	ExitInfo.uReason = pVmxTransient->uExitReason;
16038	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16039	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16040	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16041	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16042	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16043
16044	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16045	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16046	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16047	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16048	{
16049	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16050	rcStrict = VINF_SUCCESS;
16051	}
16052	return rcStrict;
16053	}
16054
16055
16056	/**
16057	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16058	*/
16059	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16060	{
16061	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16062
16063	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16064	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16065	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16066	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16067	AssertRCReturn(rc, rc);
16068
16069	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16070
16071	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16072	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
16073	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16074	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16075	{
16076	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16077	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16078	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16079	}
16080	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16081	return rcStrict;
16082	}
16083
16084
16085	/**
16086	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16087	*/
16088	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16089	{
16090	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16091
16092	/*
16093	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16094	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16095	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16096	*/
16097	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16098	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16099	hmR0VmxReadExitQualVmcs(pVmxTransient);
16100	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16101	\| CPUMCTX_EXTRN_HWVIRT
16102	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16103	AssertRCReturn(rc, rc);
16104
16105	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16106
16107	VMXVEXITINFO ExitInfo;
16108	RT_ZERO(ExitInfo);
16109	ExitInfo.uReason = pVmxTransient->uExitReason;
16110	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16111	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16112	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16113	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16114	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16115
16116	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16117	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16118	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16119	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16120	{
16121	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16122	rcStrict = VINF_SUCCESS;
16123	}
16124	return rcStrict;
16125	}
16126
16127
16128	/**
16129	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16130	*/
16131	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16132	{
16133	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16134
16135	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16136	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16137	\| CPUMCTX_EXTRN_HWVIRT
16138	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16139	AssertRCReturn(rc, rc);
16140
16141	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16142
16143	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
16144	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16145	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16146	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16147	{
16148	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16149	rcStrict = VINF_SUCCESS;
16150	}
16151	return rcStrict;
16152	}
16153
16154
16155	/**
16156	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16157	*/
16158	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16159	{
16160	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16161
16162	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16163	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16164	hmR0VmxReadExitQualVmcs(pVmxTransient);
16165	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16166	\| CPUMCTX_EXTRN_HWVIRT
16167	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16168	AssertRCReturn(rc, rc);
16169
16170	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16171
16172	VMXVEXITINFO ExitInfo;
16173	RT_ZERO(ExitInfo);
16174	ExitInfo.uReason = pVmxTransient->uExitReason;
16175	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16176	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16177	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16178	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16179
16180	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16181	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16182	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16183	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16184	{
16185	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16186	rcStrict = VINF_SUCCESS;
16187	}
16188	return rcStrict;
16189	}
16190
16191
16192	/**
16193	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16194	*/
16195	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16196	{
16197	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16198
16199	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16200	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16201	hmR0VmxReadExitQualVmcs(pVmxTransient);
16202	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16203	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16204	AssertRCReturn(rc, rc);
16205
16206	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16207
16208	VMXVEXITINFO ExitInfo;
16209	RT_ZERO(ExitInfo);
16210	ExitInfo.uReason = pVmxTransient->uExitReason;
16211	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16212	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16213	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16214	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16215
16216	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16217	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16218	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16219	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16220	{
16221	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16222	rcStrict = VINF_SUCCESS;
16223	}
16224	return rcStrict;
16225	}
16226	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16227	/** @} */
16228
16229
16230	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16231	/** @name Nested-guest VM-exit handlers.
16232	* @{
16233	*/
16234	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16235	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16236	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16237
16238	/**
16239	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16240	* Conditional VM-exit.
16241	*/
16242	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16243	{
16244	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16245
16246	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16247
16248	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16249	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16250	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16251
16252	switch (uExitIntType)
16253	{
16254	/*
16255	* Physical NMIs:
16256	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16257	*/
16258	case VMX_EXIT_INT_INFO_TYPE_NMI:
16259	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16260
16261	/*
16262	* Hardware exceptions,
16263	* Software exceptions,
16264	* Privileged software exceptions:
16265	* Figure out if the exception must be delivered to the guest or the nested-guest.
16266	*/
16267	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16268	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16269	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16270	{
16271	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16272	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16273	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16274	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16275
16276	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16277	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16278	pVmxTransient->uExitIntErrorCode);
16279	if (fIntercept)
16280	{
16281	/* Exit qualification is required for debug and page-fault exceptions. */
16282	hmR0VmxReadExitQualVmcs(pVmxTransient);
16283
16284	/*
16285	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16286	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16287	* length. However, if delivery of a software interrupt, software exception or privileged
16288	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16289	*/
16290	VMXVEXITINFO ExitInfo;
16291	RT_ZERO(ExitInfo);
16292	ExitInfo.uReason = pVmxTransient->uExitReason;
16293	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16294	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16295
16296	VMXVEXITEVENTINFO ExitEventInfo;
16297	RT_ZERO(ExitEventInfo);
16298	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16299	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16300	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16301	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16302
16303	#ifdef DEBUG_ramshankar
16304	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16305	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
16306	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16307	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16308	{
16309	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
16310	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
16311	}
16312	#endif
16313	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16314	}
16315
16316	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16317	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16318	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16319	}
16320
16321	/*
16322	* Software interrupts:
16323	* VM-exits cannot be caused by software interrupts.
16324	*
16325	* External interrupts:
16326	* This should only happen when "acknowledge external interrupts on VM-exit"
16327	* control is set. However, we never set this when executing a guest or
16328	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16329	* the guest.
16330	*/
16331	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16332	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16333	default:
16334	{
16335	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16336	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16337	}
16338	}
16339	}
16340
16341
16342	/**
16343	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16344	* Unconditional VM-exit.
16345	*/
16346	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16347	{
16348	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16349	return IEMExecVmxVmexitTripleFault(pVCpu);
16350	}
16351
16352
16353	/**
16354	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16355	*/
16356	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16357	{
16358	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16359
16360	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16361	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16362	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16363	}
16364
16365
16366	/**
16367	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16368	*/
16369	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16370	{
16371	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16372
16373	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16374	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16375	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16376	}
16377
16378
16379	/**
16380	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16381	* Unconditional VM-exit.
16382	*/
16383	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16384	{
16385	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16386
16387	hmR0VmxReadExitQualVmcs(pVmxTransient);
16388	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16389	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16390	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16391
16392	VMXVEXITINFO ExitInfo;
16393	RT_ZERO(ExitInfo);
16394	ExitInfo.uReason = pVmxTransient->uExitReason;
16395	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16396	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16397
16398	VMXVEXITEVENTINFO ExitEventInfo;
16399	RT_ZERO(ExitEventInfo);
16400	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16401	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16402	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16403	}
16404
16405
16406	/**
16407	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16408	*/
16409	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16410	{
16411	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16412
16413	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16414	{
16415	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16416	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16417	}
16418	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16419	}
16420
16421
16422	/**
16423	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16424	*/
16425	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16426	{
16427	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16428
16429	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16430	{
16431	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16432	hmR0VmxReadExitQualVmcs(pVmxTransient);
16433
16434	VMXVEXITINFO ExitInfo;
16435	RT_ZERO(ExitInfo);
16436	ExitInfo.uReason = pVmxTransient->uExitReason;
16437	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16438	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16439	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16440	}
16441	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16442	}
16443
16444
16445	/**
16446	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16447	*/
16448	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16449	{
16450	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16451
16452	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16453	{
16454	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16455	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16456	}
16457	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16458	}
16459
16460
16461	/**
16462	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16463	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16464	*/
16465	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16466	{
16467	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16468
16469	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16470	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16471
16472	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16473
16474	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16475	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16476	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16477
16478	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16479	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16480	u64VmcsField &= UINT64_C(0xffffffff);
16481
16482	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16483	{
16484	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16485	hmR0VmxReadExitQualVmcs(pVmxTransient);
16486
16487	VMXVEXITINFO ExitInfo;
16488	RT_ZERO(ExitInfo);
16489	ExitInfo.uReason = pVmxTransient->uExitReason;
16490	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16491	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16492	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16493	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16494	}
16495
16496	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16497	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16498	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16499	}
16500
16501
16502	/**
16503	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16504	*/
16505	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16506	{
16507	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16508
16509	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16510	{
16511	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16512	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16513	}
16514
16515	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16516	}
16517
16518
16519	/**
16520	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16521	* Conditional VM-exit.
16522	*/
16523	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16524	{
16525	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16526
16527	hmR0VmxReadExitQualVmcs(pVmxTransient);
16528	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16529
16530	VBOXSTRICTRC rcStrict;
16531	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16532	switch (uAccessType)
16533	{
16534	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16535	{
16536	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16537	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16538	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16539	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16540
16541	bool fIntercept;
16542	switch (iCrReg)
16543	{
16544	case 0:
16545	case 4:
16546	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16547	break;
16548
16549	case 3:
16550	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16551	break;
16552
16553	case 8:
16554	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16555	break;
16556
16557	default:
16558	fIntercept = false;
16559	break;
16560	}
16561	if (fIntercept)
16562	{
16563	VMXVEXITINFO ExitInfo;
16564	RT_ZERO(ExitInfo);
16565	ExitInfo.uReason = pVmxTransient->uExitReason;
16566	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16567	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16568	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16569	}
16570	else
16571	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16572	break;
16573	}
16574
16575	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16576	{
16577	/*
16578	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16579	* CR2 reads do not cause a VM-exit.
16580	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16581	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16582	*/
16583	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16584	if ( iCrReg == 3
16585	\|\| iCrReg == 8)
16586	{
16587	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16588	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16589	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16590	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16591	{
16592	VMXVEXITINFO ExitInfo;
16593	RT_ZERO(ExitInfo);
16594	ExitInfo.uReason = pVmxTransient->uExitReason;
16595	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16596	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16597	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16598	}
16599	else
16600	{
16601	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16602	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16603	}
16604	}
16605	else
16606	{
16607	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16608	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16609	}
16610	break;
16611	}
16612
16613	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16614	{
16615	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16616	Assert(pVmcsNstGst);
16617	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16618	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16619	if ( (uGstHostMask & X86_CR0_TS)
16620	&& (uReadShadow & X86_CR0_TS))
16621	{
16622	VMXVEXITINFO ExitInfo;
16623	RT_ZERO(ExitInfo);
16624	ExitInfo.uReason = pVmxTransient->uExitReason;
16625	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16626	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16627	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16628	}
16629	else
16630	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
16631	break;
16632	}
16633
16634	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16635	{
16636	RTGCPTR GCPtrEffDst;
16637	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16638	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16639	if (fMemOperand)
16640	{
16641	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16642	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16643	}
16644	else
16645	GCPtrEffDst = NIL_RTGCPTR;
16646
16647	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16648	{
16649	VMXVEXITINFO ExitInfo;
16650	RT_ZERO(ExitInfo);
16651	ExitInfo.uReason = pVmxTransient->uExitReason;
16652	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16653	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16654	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16655	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16656	}
16657	else
16658	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
16659	break;
16660	}
16661
16662	default:
16663	{
16664	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16665	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16666	}
16667	}
16668
16669	if (rcStrict == VINF_IEM_RAISED_XCPT)
16670	{
16671	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16672	rcStrict = VINF_SUCCESS;
16673	}
16674	return rcStrict;
16675	}
16676
16677
16678	/**
16679	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16680	* Conditional VM-exit.
16681	*/
16682	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16683	{
16684	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16685
16686	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16687	{
16688	hmR0VmxReadExitQualVmcs(pVmxTransient);
16689	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16690
16691	VMXVEXITINFO ExitInfo;
16692	RT_ZERO(ExitInfo);
16693	ExitInfo.uReason = pVmxTransient->uExitReason;
16694	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16695	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16696	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16697	}
16698	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16699	}
16700
16701
16702	/**
16703	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16704	* Conditional VM-exit.
16705	*/
16706	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16707	{
16708	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16709
16710	hmR0VmxReadExitQualVmcs(pVmxTransient);
16711
16712	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16713	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16714	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16715
16716	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16717	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16718	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16719	{
16720	/*
16721	* IN/OUT instruction:
16722	* - Provides VM-exit instruction length.
16723	*
16724	* INS/OUTS instruction:
16725	* - Provides VM-exit instruction length.
16726	* - Provides Guest-linear address.
16727	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16728	*/
16729	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16730	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16731
16732	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16733	pVmxTransient->ExitInstrInfo.u = 0;
16734	pVmxTransient->uGuestLinearAddr = 0;
16735
16736	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16737	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16738	if (fIOString)
16739	{
16740	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16741	if (fVmxInsOutsInfo)
16742	{
16743	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16744	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16745	}
16746	}
16747
16748	VMXVEXITINFO ExitInfo;
16749	RT_ZERO(ExitInfo);
16750	ExitInfo.uReason = pVmxTransient->uExitReason;
16751	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16752	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16753	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16754	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16755	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16756	}
16757	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16758	}
16759
16760
16761	/**
16762	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16763	*/
16764	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16765	{
16766	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16767
16768	uint32_t fMsrpm;
16769	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16770	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16771	else
16772	fMsrpm = VMXMSRPM_EXIT_RD;
16773
16774	if (fMsrpm & VMXMSRPM_EXIT_RD)
16775	{
16776	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16777	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16778	}
16779	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16780	}
16781
16782
16783	/**
16784	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16785	*/
16786	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16787	{
16788	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16789
16790	uint32_t fMsrpm;
16791	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16792	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16793	else
16794	fMsrpm = VMXMSRPM_EXIT_WR;
16795
16796	if (fMsrpm & VMXMSRPM_EXIT_WR)
16797	{
16798	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16799	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16800	}
16801	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16802	}
16803
16804
16805	/**
16806	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16807	*/
16808	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16809	{
16810	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16811
16812	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16813	{
16814	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16815	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16816	}
16817	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16818	}
16819
16820
16821	/**
16822	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16823	* VM-exit.
16824	*/
16825	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16826	{
16827	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16828
16829	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16830	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16831	VMXVEXITINFO ExitInfo;
16832	RT_ZERO(ExitInfo);
16833	ExitInfo.uReason = pVmxTransient->uExitReason;
16834	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16835	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16836	}
16837
16838
16839	/**
16840	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16841	*/
16842	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16843	{
16844	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16845
16846	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16847	{
16848	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16849	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16850	}
16851	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16852	}
16853
16854
16855	/**
16856	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16857	*/
16858	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16859	{
16860	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16861
16862	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16863	* PAUSE when executing a nested-guest? If it does not, we would not need
16864	* to check for the intercepts here. Just call VM-exit... */
16865
16866	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16867	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16868	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16869	{
16870	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16871	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16872	}
16873	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16874	}
16875
16876
16877	/**
16878	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16879	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16880	*/
16881	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16882	{
16883	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16884
16885	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16886	{
16887	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16888	VMXVEXITINFO ExitInfo;
16889	RT_ZERO(ExitInfo);
16890	ExitInfo.uReason = pVmxTransient->uExitReason;
16891	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16892	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16893	}
16894	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16895	}
16896
16897
16898	/**
16899	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16900	* VM-exit.
16901	*/
16902	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16903	{
16904	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16905
16906	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16907	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16908	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16909	hmR0VmxReadExitQualVmcs(pVmxTransient);
16910
16911	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16912
16913	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
16914	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
16915
16916	VMXVEXITINFO ExitInfo;
16917	RT_ZERO(ExitInfo);
16918	ExitInfo.uReason = pVmxTransient->uExitReason;
16919	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16920	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16921
16922	VMXVEXITEVENTINFO ExitEventInfo;
16923	RT_ZERO(ExitEventInfo);
16924	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16925	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16926	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
16927	}
16928
16929
16930	/**
16931	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
16932	* Conditional VM-exit.
16933	*/
16934	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16935	{
16936	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16937
16938	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
16939	hmR0VmxReadExitQualVmcs(pVmxTransient);
16940	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16941	}
16942
16943
16944	/**
16945	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
16946	* Conditional VM-exit.
16947	*/
16948	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16949	{
16950	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16951
16952	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
16953	hmR0VmxReadExitQualVmcs(pVmxTransient);
16954	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16955	}
16956
16957
16958	/**
16959	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16960	*/
16961	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16962	{
16963	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16964
16965	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16966	{
16967	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16968	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16969	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16970	}
16971	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16972	}
16973
16974
16975	/**
16976	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16977	*/
16978	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16979	{
16980	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16981
16982	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16983	{
16984	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16985	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16986	}
16987	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16988	}
16989
16990
16991	/**
16992	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16993	*/
16994	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16995	{
16996	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16997
16998	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16999	{
17000	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17001	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17002	hmR0VmxReadExitQualVmcs(pVmxTransient);
17003	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17004
17005	VMXVEXITINFO ExitInfo;
17006	RT_ZERO(ExitInfo);
17007	ExitInfo.uReason = pVmxTransient->uExitReason;
17008	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17009	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17010	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17011	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17012	}
17013	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17014	}
17015
17016
17017	/**
17018	* Nested-guest VM-exit handler for invalid-guest state
17019	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17020	*/
17021	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17022	{
17023	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17024
17025	/*
17026	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17027	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17028	* Handle it like it's in an invalid guest state of the outer guest.
17029	*
17030	* When the fast path is implemented, this should be changed to cause the corresponding
17031	* nested-guest VM-exit.
17032	*/
17033	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17034	}
17035
17036
17037	/**
17038	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17039	* and only provide the instruction length.
17040	*
17041	* Unconditional VM-exit.
17042	*/
17043	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17044	{
17045	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17046
17047	#ifdef VBOX_STRICT
17048	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17049	switch (pVmxTransient->uExitReason)
17050	{
17051	case VMX_EXIT_ENCLS:
17052	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17053	break;
17054
17055	case VMX_EXIT_VMFUNC:
17056	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17057	break;
17058	}
17059	#endif
17060
17061	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17062	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17063	}
17064
17065
17066	/**
17067	* Nested-guest VM-exit handler for instructions that provide instruction length as
17068	* well as more information.
17069	*
17070	* Unconditional VM-exit.
17071	*/
17072	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17073	{
17074	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17075
17076	#ifdef VBOX_STRICT
17077	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17078	switch (pVmxTransient->uExitReason)
17079	{
17080	case VMX_EXIT_GDTR_IDTR_ACCESS:
17081	case VMX_EXIT_LDTR_TR_ACCESS:
17082	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17083	break;
17084
17085	case VMX_EXIT_RDRAND:
17086	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17087	break;
17088
17089	case VMX_EXIT_RDSEED:
17090	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17091	break;
17092
17093	case VMX_EXIT_XSAVES:
17094	case VMX_EXIT_XRSTORS:
17095	/** @todo NSTVMX: Verify XSS-bitmap. */
17096	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17097	break;
17098
17099	case VMX_EXIT_UMWAIT:
17100	case VMX_EXIT_TPAUSE:
17101	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17102	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17103	break;
17104	}
17105	#endif
17106
17107	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17108	hmR0VmxReadExitQualVmcs(pVmxTransient);
17109	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17110
17111	VMXVEXITINFO ExitInfo;
17112	RT_ZERO(ExitInfo);
17113	ExitInfo.uReason = pVmxTransient->uExitReason;
17114	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17115	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17116	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17117	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17118	}
17119
17120	/** @} */
17121	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17122

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 81002

Download in other formats: