HMVMXR0.cpp@ 80840

Last change on this file since 80840 was 80840, checked in by vboxsync, 5 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Tighten up the CR0 guest/host mask to include reserved/unknown bits in CR0.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 708.1 KB

Line
1	/* $Id: HMVMXR0.cpp 80840 2019-09-17 05:44:30Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#ifdef VBOX_WITH_REM
39	# include <VBox/vmm/rem.h>
40	#endif
41	#include "HMInternal.h"
42	#include <VBox/vmm/vmcc.h>
43	#include <VBox/vmm/hmvmxinline.h>
44	#include "HMVMXR0.h"
45	#include "dtrace/VBoxVMM.h"
46
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
50	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
51	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
52	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
53	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
54	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
55	# define HMVMX_ALWAYS_TRAP_PF
56	# define HMVMX_ALWAYS_FLUSH_TLB
57	# define HMVMX_ALWAYS_SWAP_EFER
58	#endif
59
60
61	/*********************************************************************************************************************************
62	* Defined Constants And Macros *
63	*********************************************************************************************************************************/
64	/** Use the function table. */
65	#define HMVMX_USE_FUNCTION_TABLE
66
67	/** Determine which tagged-TLB flush handler to use. */
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
70	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
71	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
72
73	/**
74	* Flags to skip redundant reads of some common VMCS fields that are not part of
75	* the guest-CPU or VCPU state but are needed while handling VM-exits.
76	*/
77	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
78	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
79	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
80	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
81	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
82	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
83	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
84	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
85	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
86	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
87
88	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
89	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
90	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
91	\| HMVMX_READ_EXIT_INSTR_LEN \
92	\| HMVMX_READ_IDT_VECTORING_INFO \
93	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
94
95	/** Assert that all the given fields have been read from the VMCS. */
96	#ifdef VBOX_STRICT
97	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
98	do { \
99	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
100	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); \
101	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
102	} while (0)
103	#else
104	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
105	#endif
106
107	/**
108	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
109	* guest using hardware-assisted VMX.
110	*
111	* This excludes state like GPRs (other than RSP) which are always are
112	* swapped and restored across the world-switch and also registers like EFER,
113	* MSR which cannot be modified by the guest without causing a VM-exit.
114	*/
115	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
116	\| CPUMCTX_EXTRN_RFLAGS \
117	\| CPUMCTX_EXTRN_RSP \
118	\| CPUMCTX_EXTRN_SREG_MASK \
119	\| CPUMCTX_EXTRN_TABLE_MASK \
120	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
121	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
122	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
123	\| CPUMCTX_EXTRN_TSC_AUX \
124	\| CPUMCTX_EXTRN_OTHER_MSRS \
125	\| CPUMCTX_EXTRN_CR0 \
126	\| CPUMCTX_EXTRN_CR3 \
127	\| CPUMCTX_EXTRN_CR4 \
128	\| CPUMCTX_EXTRN_DR7 \
129	\| CPUMCTX_EXTRN_HWVIRT \
130	\| CPUMCTX_EXTRN_HM_VMX_MASK)
131
132	/**
133	* Exception bitmap mask for real-mode guests (real-on-v86).
134	*
135	* We need to intercept all exceptions manually except:
136	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
137	* due to bugs in Intel CPUs.
138	* - \#PF need not be intercepted even in real-mode if we have nested paging
139	* support.
140	*/
141	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
142	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
143	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
144	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
145	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
146	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
147	\| RT_BIT(X86_XCPT_XF))
148
149	/** Maximum VM-instruction error number. */
150	#define HMVMX_INSTR_ERROR_MAX 28
151
152	/** Profiling macro. */
153	#ifdef HM_PROFILE_EXIT_DISPATCH
154	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
156	#else
157	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
158	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
159	#endif
160
161	/** Assert that preemption is disabled or covered by thread-context hooks. */
162	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
163	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
164
165	/** Assert that we haven't migrated CPUs when thread-context hooks are not
166	* used. */
167	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
168	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
169	("Illegal migration! Entered on CPU %u Current %u\n", \
170	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
171
172	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
173	* context. */
174	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
175	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
176	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
177
178	/** Log the VM-exit reason with an easily visible marker to identify it in a
179	* potential sea of logging data. */
180	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
181	do { \
182	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
183	HMGetVmxExitName(a_uExitReason))); \
184	} while (0) \
185
186
187	/*********************************************************************************************************************************
188	* Structures and Typedefs *
189	*********************************************************************************************************************************/
190	/**
191	* VMX per-VCPU transient state.
192	*
193	* A state structure for holding miscellaneous information across
194	* VMX non-root operation and restored after the transition.
195	*/
196	typedef struct VMXTRANSIENT
197	{
198	/** The host's rflags/eflags. */
199	RTCCUINTREG fEFlags;
200
201	/** The guest's TPR value used for TPR shadowing. */
202	uint8_t u8GuestTpr;
203	/** Alignment. */
204	uint8_t abAlignment0[7];
205
206	/** The basic VM-exit reason. */
207	uint16_t uExitReason;
208	/** Alignment. */
209	uint16_t u16Alignment0;
210	/** The VM-exit interruption error code. */
211	uint32_t uExitIntErrorCode;
212	/** The VM-exit exit code qualification. */
213	uint64_t uExitQual;
214	/** The Guest-linear address. */
215	uint64_t uGuestLinearAddr;
216	/** The Guest-physical address. */
217	uint64_t uGuestPhysicalAddr;
218	/** The Guest pending-debug exceptions. */
219	uint64_t uGuestPendingDbgXcpts;
220
221	/** The VM-exit interruption-information field. */
222	uint32_t uExitIntInfo;
223	/** The VM-exit instruction-length field. */
224	uint32_t cbExitInstr;
225	/** The VM-exit instruction-information field. */
226	VMXEXITINSTRINFO ExitInstrInfo;
227	/** Whether the VM-entry failed or not. */
228	bool fVMEntryFailed;
229	/** Whether we are currently executing a nested-guest. */
230	bool fIsNestedGuest;
231	/** Alignment. */
232	uint8_t abAlignment1[2];
233
234	/** The VM-entry interruption-information field. */
235	uint32_t uEntryIntInfo;
236	/** The VM-entry exception error code field. */
237	uint32_t uEntryXcptErrorCode;
238	/** The VM-entry instruction length field. */
239	uint32_t cbEntryInstr;
240
241	/** IDT-vectoring information field. */
242	uint32_t uIdtVectoringInfo;
243	/** IDT-vectoring error code. */
244	uint32_t uIdtVectoringErrorCode;
245
246	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
247	uint32_t fVmcsFieldsRead;
248
249	/** Whether the guest debug state was active at the time of VM-exit. */
250	bool fWasGuestDebugStateActive;
251	/** Whether the hyper debug state was active at the time of VM-exit. */
252	bool fWasHyperDebugStateActive;
253	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
254	bool fUpdatedTscOffsettingAndPreemptTimer;
255	/** Whether the VM-exit was caused by a page-fault during delivery of a
256	* contributory exception or a page-fault. */
257	bool fVectoringDoublePF;
258	/** Whether the VM-exit was caused by a page-fault during delivery of an
259	* external interrupt or NMI. */
260	bool fVectoringPF;
261	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
262	* area after VM-exit. */
263	bool fRemoveTscAuxMsr;
264	bool afAlignment0[2];
265
266	/** The VMCS info. object. */
267	PVMXVMCSINFO pVmcsInfo;
268	} VMXTRANSIENT;
269	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
271	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
272	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
273	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
274	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
275	/** Pointer to VMX transient state. */
276	typedef VMXTRANSIENT *PVMXTRANSIENT;
277	/** Pointer to a const VMX transient state. */
278	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
279
280	/**
281	* Memory operand read or write access.
282	*/
283	typedef enum VMXMEMACCESS
284	{
285	VMXMEMACCESS_READ = 0,
286	VMXMEMACCESS_WRITE = 1
287	} VMXMEMACCESS;
288
289	/**
290	* VMX VM-exit handler.
291	*
292	* @returns Strict VBox status code (i.e. informational status codes too).
293	* @param pVCpu The cross context virtual CPU structure.
294	* @param pVmxTransient The VMX-transient structure.
295	*/
296	#ifndef HMVMX_USE_FUNCTION_TABLE
297	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
298	#else
299	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
300	/** Pointer to VM-exit handler. */
301	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
302	#endif
303
304	/**
305	* VMX VM-exit handler, non-strict status code.
306	*
307	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
308	*
309	* @returns VBox status code, no informational status code returned.
310	* @param pVCpu The cross context virtual CPU structure.
311	* @param pVmxTransient The VMX-transient structure.
312	*
313	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
314	* use of that status code will be replaced with VINF_EM_SOMETHING
315	* later when switching over to IEM.
316	*/
317	#ifndef HMVMX_USE_FUNCTION_TABLE
318	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
319	#else
320	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
321	#endif
322
323
324	/*********************************************************************************************************************************
325	* Internal Functions *
326	*********************************************************************************************************************************/
327	#ifndef HMVMX_USE_FUNCTION_TABLE
328	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
329	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
330	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
331	#else
332	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
333	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
334	#endif
335	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
336	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
337	#endif
338
339	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
340
341	/** @name VM-exit handler prototypes.
342	* @{
343	*/
344	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
345	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
346	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
347	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
348	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
349	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
350	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
351	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
352	static FNVMXEXITHANDLER hmR0VmxExitHlt;
353	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
354	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
355	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
356	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
357	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
358	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
359	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
360	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
361	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
362	static FNVMXEXITHANDLER hmR0VmxExitVmread;
363	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
364	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
365	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
366	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
367	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
368	#endif
369	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
370	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
371	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
372	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
373	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
374	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
375	static FNVMXEXITHANDLER hmR0VmxExitMwait;
376	static FNVMXEXITHANDLER hmR0VmxExitMtf;
377	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
378	static FNVMXEXITHANDLER hmR0VmxExitPause;
379	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
380	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
381	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
382	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
383	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
384	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
385	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
386	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
387	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
391	/** @} */
392
393	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
394	/** @name Nested-guest VM-exit handler prototypes.
395	* @{
396	*/
397	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
398	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
400	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
401	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
402	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
403	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
404	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
405	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
406	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
407	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
408	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
409	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
410	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
411	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
412	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
413	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
414	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
415	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
417	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
418	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
419	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
420	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
422	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
423	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
424	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
425	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
426	/** @} */
427	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
428
429
430	/*********************************************************************************************************************************
431	* Global Variables *
432	*********************************************************************************************************************************/
433	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
434	/**
435	* Array of all VMCS fields.
436	* Any fields added to the VT-x spec. should be added here.
437	*
438	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
439	* of nested-guests.
440	*/
441	static const uint32_t g_aVmcsFields[] =
442	{
443	/* 16-bit control fields. */
444	VMX_VMCS16_VPID,
445	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
446	VMX_VMCS16_EPTP_INDEX,
447
448	/* 16-bit guest-state fields. */
449	VMX_VMCS16_GUEST_ES_SEL,
450	VMX_VMCS16_GUEST_CS_SEL,
451	VMX_VMCS16_GUEST_SS_SEL,
452	VMX_VMCS16_GUEST_DS_SEL,
453	VMX_VMCS16_GUEST_FS_SEL,
454	VMX_VMCS16_GUEST_GS_SEL,
455	VMX_VMCS16_GUEST_LDTR_SEL,
456	VMX_VMCS16_GUEST_TR_SEL,
457	VMX_VMCS16_GUEST_INTR_STATUS,
458	VMX_VMCS16_GUEST_PML_INDEX,
459
460	/* 16-bits host-state fields. */
461	VMX_VMCS16_HOST_ES_SEL,
462	VMX_VMCS16_HOST_CS_SEL,
463	VMX_VMCS16_HOST_SS_SEL,
464	VMX_VMCS16_HOST_DS_SEL,
465	VMX_VMCS16_HOST_FS_SEL,
466	VMX_VMCS16_HOST_GS_SEL,
467	VMX_VMCS16_HOST_TR_SEL,
468
469	/* 64-bit control fields. */
470	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
471	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
472	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
473	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
474	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
475	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
476	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
477	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
478	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
479	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
480	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
481	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
482	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
483	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
484	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
485	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
486	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
487	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
488	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
489	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
490	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
491	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
492	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
493	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
494	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
495	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
496	VMX_VMCS64_CTRL_EPTP_FULL,
497	VMX_VMCS64_CTRL_EPTP_HIGH,
498	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
499	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
500	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
501	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
502	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
503	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
504	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
505	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
506	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
507	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
508	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
509	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
510	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
511	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
512	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
513	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
514	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
515	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
516	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
517	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
518	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
519	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
520
521	/* 64-bit read-only data fields. */
522	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
523	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
524
525	/* 64-bit guest-state fields. */
526	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
527	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
528	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
529	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
530	VMX_VMCS64_GUEST_PAT_FULL,
531	VMX_VMCS64_GUEST_PAT_HIGH,
532	VMX_VMCS64_GUEST_EFER_FULL,
533	VMX_VMCS64_GUEST_EFER_HIGH,
534	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
535	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
536	VMX_VMCS64_GUEST_PDPTE0_FULL,
537	VMX_VMCS64_GUEST_PDPTE0_HIGH,
538	VMX_VMCS64_GUEST_PDPTE1_FULL,
539	VMX_VMCS64_GUEST_PDPTE1_HIGH,
540	VMX_VMCS64_GUEST_PDPTE2_FULL,
541	VMX_VMCS64_GUEST_PDPTE2_HIGH,
542	VMX_VMCS64_GUEST_PDPTE3_FULL,
543	VMX_VMCS64_GUEST_PDPTE3_HIGH,
544	VMX_VMCS64_GUEST_BNDCFGS_FULL,
545	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
546
547	/* 64-bit host-state fields. */
548	VMX_VMCS64_HOST_PAT_FULL,
549	VMX_VMCS64_HOST_PAT_HIGH,
550	VMX_VMCS64_HOST_EFER_FULL,
551	VMX_VMCS64_HOST_EFER_HIGH,
552	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
553	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
554
555	/* 32-bit control fields. */
556	VMX_VMCS32_CTRL_PIN_EXEC,
557	VMX_VMCS32_CTRL_PROC_EXEC,
558	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
559	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
560	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
561	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
562	VMX_VMCS32_CTRL_EXIT,
563	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
564	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
565	VMX_VMCS32_CTRL_ENTRY,
566	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
567	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
568	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
569	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
570	VMX_VMCS32_CTRL_TPR_THRESHOLD,
571	VMX_VMCS32_CTRL_PROC_EXEC2,
572	VMX_VMCS32_CTRL_PLE_GAP,
573	VMX_VMCS32_CTRL_PLE_WINDOW,
574
575	/* 32-bits read-only fields. */
576	VMX_VMCS32_RO_VM_INSTR_ERROR,
577	VMX_VMCS32_RO_EXIT_REASON,
578	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
579	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
580	VMX_VMCS32_RO_IDT_VECTORING_INFO,
581	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
582	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
583	VMX_VMCS32_RO_EXIT_INSTR_INFO,
584
585	/* 32-bit guest-state fields. */
586	VMX_VMCS32_GUEST_ES_LIMIT,
587	VMX_VMCS32_GUEST_CS_LIMIT,
588	VMX_VMCS32_GUEST_SS_LIMIT,
589	VMX_VMCS32_GUEST_DS_LIMIT,
590	VMX_VMCS32_GUEST_FS_LIMIT,
591	VMX_VMCS32_GUEST_GS_LIMIT,
592	VMX_VMCS32_GUEST_LDTR_LIMIT,
593	VMX_VMCS32_GUEST_TR_LIMIT,
594	VMX_VMCS32_GUEST_GDTR_LIMIT,
595	VMX_VMCS32_GUEST_IDTR_LIMIT,
596	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
597	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
598	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
599	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
600	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
602	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
603	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
604	VMX_VMCS32_GUEST_INT_STATE,
605	VMX_VMCS32_GUEST_ACTIVITY_STATE,
606	VMX_VMCS32_GUEST_SMBASE,
607	VMX_VMCS32_GUEST_SYSENTER_CS,
608	VMX_VMCS32_PREEMPT_TIMER_VALUE,
609
610	/* 32-bit host-state fields. */
611	VMX_VMCS32_HOST_SYSENTER_CS,
612
613	/* Natural-width control fields. */
614	VMX_VMCS_CTRL_CR0_MASK,
615	VMX_VMCS_CTRL_CR4_MASK,
616	VMX_VMCS_CTRL_CR0_READ_SHADOW,
617	VMX_VMCS_CTRL_CR4_READ_SHADOW,
618	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
619	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
620	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
621	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
622
623	/* Natural-width read-only data fields. */
624	VMX_VMCS_RO_EXIT_QUALIFICATION,
625	VMX_VMCS_RO_IO_RCX,
626	VMX_VMCS_RO_IO_RSI,
627	VMX_VMCS_RO_IO_RDI,
628	VMX_VMCS_RO_IO_RIP,
629	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
630
631	/* Natural-width guest-state field */
632	VMX_VMCS_GUEST_CR0,
633	VMX_VMCS_GUEST_CR3,
634	VMX_VMCS_GUEST_CR4,
635	VMX_VMCS_GUEST_ES_BASE,
636	VMX_VMCS_GUEST_CS_BASE,
637	VMX_VMCS_GUEST_SS_BASE,
638	VMX_VMCS_GUEST_DS_BASE,
639	VMX_VMCS_GUEST_FS_BASE,
640	VMX_VMCS_GUEST_GS_BASE,
641	VMX_VMCS_GUEST_LDTR_BASE,
642	VMX_VMCS_GUEST_TR_BASE,
643	VMX_VMCS_GUEST_GDTR_BASE,
644	VMX_VMCS_GUEST_IDTR_BASE,
645	VMX_VMCS_GUEST_DR7,
646	VMX_VMCS_GUEST_RSP,
647	VMX_VMCS_GUEST_RIP,
648	VMX_VMCS_GUEST_RFLAGS,
649	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
650	VMX_VMCS_GUEST_SYSENTER_ESP,
651	VMX_VMCS_GUEST_SYSENTER_EIP,
652
653	/* Natural-width host-state fields */
654	VMX_VMCS_HOST_CR0,
655	VMX_VMCS_HOST_CR3,
656	VMX_VMCS_HOST_CR4,
657	VMX_VMCS_HOST_FS_BASE,
658	VMX_VMCS_HOST_GS_BASE,
659	VMX_VMCS_HOST_TR_BASE,
660	VMX_VMCS_HOST_GDTR_BASE,
661	VMX_VMCS_HOST_IDTR_BASE,
662	VMX_VMCS_HOST_SYSENTER_ESP,
663	VMX_VMCS_HOST_SYSENTER_EIP,
664	VMX_VMCS_HOST_RSP,
665	VMX_VMCS_HOST_RIP
666	};
667	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
668
669	static const uint32_t g_aVmcsSegBase[] =
670	{
671	VMX_VMCS_GUEST_ES_BASE,
672	VMX_VMCS_GUEST_CS_BASE,
673	VMX_VMCS_GUEST_SS_BASE,
674	VMX_VMCS_GUEST_DS_BASE,
675	VMX_VMCS_GUEST_FS_BASE,
676	VMX_VMCS_GUEST_GS_BASE
677	};
678	static const uint32_t g_aVmcsSegSel[] =
679	{
680	VMX_VMCS16_GUEST_ES_SEL,
681	VMX_VMCS16_GUEST_CS_SEL,
682	VMX_VMCS16_GUEST_SS_SEL,
683	VMX_VMCS16_GUEST_DS_SEL,
684	VMX_VMCS16_GUEST_FS_SEL,
685	VMX_VMCS16_GUEST_GS_SEL
686	};
687	static const uint32_t g_aVmcsSegLimit[] =
688	{
689	VMX_VMCS32_GUEST_ES_LIMIT,
690	VMX_VMCS32_GUEST_CS_LIMIT,
691	VMX_VMCS32_GUEST_SS_LIMIT,
692	VMX_VMCS32_GUEST_DS_LIMIT,
693	VMX_VMCS32_GUEST_FS_LIMIT,
694	VMX_VMCS32_GUEST_GS_LIMIT
695	};
696	static const uint32_t g_aVmcsSegAttr[] =
697	{
698	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
699	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
700	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
701	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
702	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
703	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
704	};
705	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
706	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
707	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
708	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
709
710	#ifdef HMVMX_USE_FUNCTION_TABLE
711	/**
712	* VMX_EXIT dispatch table.
713	*/
714	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
715	{
716	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
717	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
718	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
719	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
720	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
721	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
722	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
723	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
724	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
725	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
726	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
727	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
728	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
729	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
730	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
731	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
732	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
733	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
734	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
735	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
736	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
737	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
738	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
739	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
740	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
741	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
742	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
743	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
744	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
745	#else
746	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
747	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
748	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
749	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
750	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
751	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
752	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
753	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
754	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
755	#endif
756	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
757	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
758	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
759	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
760	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
761	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
762	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
763	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
764	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
765	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
766	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
767	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
768	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
769	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
770	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
771	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
772	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
773	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
774	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
775	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
776	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
777	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
778	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
779	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
780	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
781	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
782	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
783	#else
784	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
785	#endif
786	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
787	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
788	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
789	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
790	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
791	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
792	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
793	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
794	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
795	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
796	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
797	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
798	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
799	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
800	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
801	};
802	#endif /* HMVMX_USE_FUNCTION_TABLE */
803
804	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
805	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
806	{
807	/* 0 */ "(Not Used)",
808	/* 1 */ "VMCALL executed in VMX root operation.",
809	/* 2 */ "VMCLEAR with invalid physical address.",
810	/* 3 */ "VMCLEAR with VMXON pointer.",
811	/* 4 */ "VMLAUNCH with non-clear VMCS.",
812	/* 5 */ "VMRESUME with non-launched VMCS.",
813	/* 6 */ "VMRESUME after VMXOFF",
814	/* 7 */ "VM-entry with invalid control fields.",
815	/* 8 */ "VM-entry with invalid host state fields.",
816	/* 9 */ "VMPTRLD with invalid physical address.",
817	/* 10 */ "VMPTRLD with VMXON pointer.",
818	/* 11 */ "VMPTRLD with incorrect revision identifier.",
819	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
820	/* 13 */ "VMWRITE to read-only VMCS component.",
821	/* 14 */ "(Not Used)",
822	/* 15 */ "VMXON executed in VMX root operation.",
823	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
824	/* 17 */ "VM-entry with non-launched executing VMCS.",
825	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
826	/* 19 */ "VMCALL with non-clear VMCS.",
827	/* 20 */ "VMCALL with invalid VM-exit control fields.",
828	/* 21 */ "(Not Used)",
829	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
830	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
831	/* 24 */ "VMCALL with invalid SMM-monitor features.",
832	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
833	/* 26 */ "VM-entry with events blocked by MOV SS.",
834	/* 27 */ "(Not Used)",
835	/* 28 */ "Invalid operand to INVEPT/INVVPID."
836	};
837	#endif /* VBOX_STRICT && LOG_ENABLED */
838
839
840	/**
841	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
842	*
843	* Any bit set in this mask is owned by the host/hypervisor and would cause a
844	* VM-exit when modified by the guest.
845	*
846	* @returns The static CR0 guest/host mask.
847	* @param pVCpu The cross context virtual CPU structure.
848	*/
849	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPUCC pVCpu)
850	{
851	/*
852	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
853	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
854	*
855	* Furthermore, modifications to any bits that are reserved/unspecified currently
856	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
857	* when future CPUs specify and use currently reserved/unspecified bits.
858	*/
859	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
860	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
861	* and @bugref{6944}. */
862	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
863	return ( X86_CR0_PE
864	\| X86_CR0_NE
865	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
866	\| X86_CR0_PG
867	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
868	}
869
870
871	/**
872	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
873	*
874	* Any bit set in this mask is owned by the host/hypervisor and would cause a
875	* VM-exit when modified by the guest.
876	*
877	* @returns The static CR4 guest/host mask.
878	* @param pVCpu The cross context virtual CPU structure.
879	*/
880	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPUCC pVCpu)
881	{
882	/*
883	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
884	* these bits are reserved on hardware that does not support them. Since the
885	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
886	* these bits and handle it depending on whether we expose them to the guest.
887	*/
888	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
889	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
890	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
891	return ( X86_CR4_VMXE
892	\| X86_CR4_VME
893	\| X86_CR4_PAE
894	\| X86_CR4_PGE
895	\| X86_CR4_PSE
896	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
897	\| (fPcid ? X86_CR4_PCIDE : 0));
898	}
899
900
901	/**
902	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
903	* area.
904	*
905	* @returns @c true if it's different, @c false otherwise.
906	* @param pVmcsInfo The VMCS info. object.
907	*/
908	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
909	{
910	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
911	&& pVmcsInfo->pvGuestMsrStore);
912	}
913
914
915	/**
916	* Sets the given Processor-based VM-execution controls.
917	*
918	* @param pVmxTransient The VMX-transient structure.
919	* @param uProcCtls The Processor-based VM-execution controls to set.
920	*/
921	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
922	{
923	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
924	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
925	{
926	pVmcsInfo->u32ProcCtls \|= uProcCtls;
927	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
928	AssertRC(rc);
929	}
930	}
931
932
933	/**
934	* Removes the given Processor-based VM-execution controls.
935	*
936	* @param pVCpu The cross context virtual CPU structure.
937	* @param pVmxTransient The VMX-transient structure.
938	* @param uProcCtls The Processor-based VM-execution controls to remove.
939	*
940	* @remarks When executing a nested-guest, this will not remove any of the specified
941	* controls if the nested hypervisor has set any one of them.
942	*/
943	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
944	{
945	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
946	if (pVmcsInfo->u32ProcCtls & uProcCtls)
947	{
948	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
949	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
950	? true
951	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
952	#else
953	NOREF(pVCpu);
954	bool const fRemoveCtls = true;
955	#endif
956	if (fRemoveCtls)
957	{
958	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
959	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
960	AssertRC(rc);
961	}
962	}
963	}
964
965
966	/**
967	* Sets the TSC offset for the current VMCS.
968	*
969	* @param uTscOffset The TSC offset to set.
970	* @param pVmcsInfo The VMCS info. object.
971	*/
972	static void hmR0VmxSetTscOffsetVmcs(PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
973	{
974	if (pVmcsInfo->u64TscOffset != uTscOffset)
975	{
976	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
977	AssertRC(rc);
978	pVmcsInfo->u64TscOffset = uTscOffset;
979	}
980	}
981
982
983	/**
984	* Adds one or more exceptions to the exception bitmap and commits it to the current
985	* VMCS.
986	*
987	* @param pVmxTransient The VMX-transient structure.
988	* @param uXcptMask The exception(s) to add.
989	*/
990	static void hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
991	{
992	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
993	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
994	if ((uXcptBitmap & uXcptMask) != uXcptMask)
995	{
996	uXcptBitmap \|= uXcptMask;
997	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
998	AssertRC(rc);
999	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1000	}
1001	}
1002
1003
1004	/**
1005	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1006	*
1007	* @param pVmxTransient The VMX-transient structure.
1008	* @param uXcpt The exception to add.
1009	*/
1010	static void hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1011	{
1012	Assert(uXcpt <= X86_XCPT_LAST);
1013	hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1014	}
1015
1016
1017	/**
1018	* Remove one or more exceptions from the exception bitmap and commits it to the
1019	* current VMCS.
1020	*
1021	* This takes care of not removing the exception intercept if a nested-guest
1022	* requires the exception to be intercepted.
1023	*
1024	* @returns VBox status code.
1025	* @param pVCpu The cross context virtual CPU structure.
1026	* @param pVmxTransient The VMX-transient structure.
1027	* @param uXcptMask The exception(s) to remove.
1028	*/
1029	static int hmR0VmxRemoveXcptInterceptMask(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1030	{
1031	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1032	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1033	if (u32XcptBitmap & uXcptMask)
1034	{
1035	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1036	if (!pVmxTransient->fIsNestedGuest)
1037	{ /* likely */ }
1038	else
1039	{
1040	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1041	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1042	}
1043	#endif
1044	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1045	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1046	\| RT_BIT(X86_XCPT_DE)
1047	\| RT_BIT(X86_XCPT_NM)
1048	\| RT_BIT(X86_XCPT_TS)
1049	\| RT_BIT(X86_XCPT_UD)
1050	\| RT_BIT(X86_XCPT_NP)
1051	\| RT_BIT(X86_XCPT_SS)
1052	\| RT_BIT(X86_XCPT_GP)
1053	\| RT_BIT(X86_XCPT_PF)
1054	\| RT_BIT(X86_XCPT_MF));
1055	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1056	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1057	#endif
1058	if (uXcptMask)
1059	{
1060	/* Validate we are not removing any essential exception intercepts. */
1061	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
1062	NOREF(pVCpu);
1063	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1064	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1065
1066	/* Remove it from the exception bitmap. */
1067	u32XcptBitmap &= ~uXcptMask;
1068
1069	/* Commit and update the cache if necessary. */
1070	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1071	{
1072	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1073	AssertRC(rc);
1074	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1075	}
1076	}
1077	}
1078	return VINF_SUCCESS;
1079	}
1080
1081
1082	/**
1083	* Remove an exceptions from the exception bitmap and commits it to the current
1084	* VMCS.
1085	*
1086	* @returns VBox status code.
1087	* @param pVCpu The cross context virtual CPU structure.
1088	* @param pVmxTransient The VMX-transient structure.
1089	* @param uXcpt The exception to remove.
1090	*/
1091	static int hmR0VmxRemoveXcptIntercept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1092	{
1093	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1094	}
1095
1096
1097	/**
1098	* Loads the VMCS specified by the VMCS info. object.
1099	*
1100	* @returns VBox status code.
1101	* @param pVmcsInfo The VMCS info. object.
1102	*
1103	* @remarks Can be called with interrupts disabled.
1104	*/
1105	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1106	{
1107	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1108	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1109
1110	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1111	if (RT_SUCCESS(rc))
1112	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1113	return rc;
1114	}
1115
1116
1117	/**
1118	* Clears the VMCS specified by the VMCS info. object.
1119	*
1120	* @returns VBox status code.
1121	* @param pVmcsInfo The VMCS info. object.
1122	*
1123	* @remarks Can be called with interrupts disabled.
1124	*/
1125	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1126	{
1127	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1128	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1129
1130	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1131	if (RT_SUCCESS(rc))
1132	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1133	return rc;
1134	}
1135
1136
1137	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1138	/**
1139	* Loads the shadow VMCS specified by the VMCS info. object.
1140	*
1141	* @returns VBox status code.
1142	* @param pVmcsInfo The VMCS info. object.
1143	*
1144	* @remarks Can be called with interrupts disabled.
1145	*/
1146	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1147	{
1148	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1149	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1150
1151	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1152	if (RT_SUCCESS(rc))
1153	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1154	return rc;
1155	}
1156
1157
1158	/**
1159	* Clears the shadow VMCS specified by the VMCS info. object.
1160	*
1161	* @returns VBox status code.
1162	* @param pVmcsInfo The VMCS info. object.
1163	*
1164	* @remarks Can be called with interrupts disabled.
1165	*/
1166	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1167	{
1168	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1169	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1170
1171	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1172	if (RT_SUCCESS(rc))
1173	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1174	return rc;
1175	}
1176
1177
1178	/**
1179	* Switches from and to the specified VMCSes.
1180	*
1181	* @returns VBox status code.
1182	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1183	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1184	*
1185	* @remarks Called with interrupts disabled.
1186	*/
1187	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1188	{
1189	/*
1190	* Clear the VMCS we are switching out if it has not already been cleared.
1191	* This will sync any CPU internal data back to the VMCS.
1192	*/
1193	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1194	{
1195	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1196	if (RT_SUCCESS(rc))
1197	{
1198	/*
1199	* The shadow VMCS, if any, would not be active at this point since we
1200	* would have cleared it while importing the virtual hardware-virtualization
1201	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1202	* clear the shadow VMCS here, just assert for safety.
1203	*/
1204	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1205	}
1206	else
1207	return rc;
1208	}
1209
1210	/*
1211	* Clear the VMCS we are switching to if it has not already been cleared.
1212	* This will initialize the VMCS launch state to "clear" required for loading it.
1213	*
1214	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1215	*/
1216	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1217	{
1218	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1219	if (RT_SUCCESS(rc))
1220	{ /* likely */ }
1221	else
1222	return rc;
1223	}
1224
1225	/*
1226	* Finally, load the VMCS we are switching to.
1227	*/
1228	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1229	}
1230
1231
1232	/**
1233	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1234	* caller.
1235	*
1236	* @returns VBox status code.
1237	* @param pVCpu The cross context virtual CPU structure.
1238	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1239	* true) or guest VMCS (pass false).
1240	*/
1241	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1242	{
1243	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1244	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1245
1246	PVMXVMCSINFO pVmcsInfoFrom;
1247	PVMXVMCSINFO pVmcsInfoTo;
1248	if (fSwitchToNstGstVmcs)
1249	{
1250	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1251	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1252	}
1253	else
1254	{
1255	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1256	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1257	}
1258
1259	/*
1260	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1261	* preemption hook code path acquires the current VMCS.
1262	*/
1263	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1264
1265	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1266	if (RT_SUCCESS(rc))
1267	{
1268	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1269
1270	/*
1271	* If we are switching to a VMCS that was executed on a different host CPU or was
1272	* never executed before, flag that we need to export the host state before executing
1273	* guest/nested-guest code using hardware-assisted VMX.
1274	*
1275	* This could probably be done in a preemptible context since the preemption hook
1276	* will flag the necessary change in host context. However, since preemption is
1277	* already disabled and to avoid making assumptions about host specific code in
1278	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1279	* disabled.
1280	*/
1281	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1282	{ /* likely */ }
1283	else
1284	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1285
1286	ASMSetFlags(fEFlags);
1287
1288	/*
1289	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1290	* flag that we need to update the host MSR values there. Even if we decide in the
1291	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1292	* if its content differs, we would have to update the host MSRs anyway.
1293	*/
1294	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1295	}
1296	else
1297	ASMSetFlags(fEFlags);
1298	return rc;
1299	}
1300	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1301
1302
1303	/**
1304	* Updates the VM's last error record.
1305	*
1306	* If there was a VMX instruction error, reads the error data from the VMCS and
1307	* updates VCPU's last error record as well.
1308	*
1309	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1310	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1311	* VERR_VMX_INVALID_VMCS_FIELD.
1312	* @param rc The error code.
1313	*/
1314	static void hmR0VmxUpdateErrorRecord(PVMCPUCC pVCpu, int rc)
1315	{
1316	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1317	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1318	{
1319	AssertPtrReturnVoid(pVCpu);
1320	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1321	}
1322	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1323	}
1324
1325
1326	#ifdef VBOX_STRICT
1327	/**
1328	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1329	* transient structure.
1330	*
1331	* @param pVmxTransient The VMX-transient structure.
1332	*/
1333	DECLINLINE(void) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1334	{
1335	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1336	AssertRC(rc);
1337	}
1338
1339
1340	/**
1341	* Reads the VM-entry exception error code field from the VMCS into
1342	* the VMX transient structure.
1343	*
1344	* @param pVmxTransient The VMX-transient structure.
1345	*/
1346	DECLINLINE(void) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1347	{
1348	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1349	AssertRC(rc);
1350	}
1351
1352
1353	/**
1354	* Reads the VM-entry exception error code field from the VMCS into
1355	* the VMX transient structure.
1356	*
1357	* @param pVmxTransient The VMX-transient structure.
1358	*/
1359	DECLINLINE(void) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1360	{
1361	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1362	AssertRC(rc);
1363	}
1364	#endif /* VBOX_STRICT */
1365
1366
1367	/**
1368	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1369	* transient structure.
1370	*
1371	* @param pVmxTransient The VMX-transient structure.
1372	*/
1373	DECLINLINE(void) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1374	{
1375	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1376	{
1377	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1378	AssertRC(rc);
1379	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1380	}
1381	}
1382
1383
1384	/**
1385	* Reads the VM-exit interruption error code from the VMCS into the VMX
1386	* transient structure.
1387	*
1388	* @param pVmxTransient The VMX-transient structure.
1389	*/
1390	DECLINLINE(void) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1391	{
1392	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1393	{
1394	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1395	AssertRC(rc);
1396	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1397	}
1398	}
1399
1400
1401	/**
1402	* Reads the VM-exit instruction length field from the VMCS into the VMX
1403	* transient structure.
1404	*
1405	* @param pVmxTransient The VMX-transient structure.
1406	*/
1407	DECLINLINE(void) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1408	{
1409	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1410	{
1411	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1412	AssertRC(rc);
1413	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1414	}
1415	}
1416
1417
1418	/**
1419	* Reads the VM-exit instruction-information field from the VMCS into
1420	* the VMX transient structure.
1421	*
1422	* @param pVmxTransient The VMX-transient structure.
1423	*/
1424	DECLINLINE(void) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1425	{
1426	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1427	{
1428	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1429	AssertRC(rc);
1430	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1431	}
1432	}
1433
1434
1435	/**
1436	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1437	*
1438	* @param pVmxTransient The VMX-transient structure.
1439	*/
1440	DECLINLINE(void) hmR0VmxReadExitQualVmcs(PVMXTRANSIENT pVmxTransient)
1441	{
1442	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1443	{
1444	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1445	AssertRC(rc);
1446	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1447	}
1448	}
1449
1450
1451	/**
1452	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1453	*
1454	* @param pVmxTransient The VMX-transient structure.
1455	*/
1456	DECLINLINE(void) hmR0VmxReadGuestLinearAddrVmcs(PVMXTRANSIENT pVmxTransient)
1457	{
1458	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1459	{
1460	int rc = VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1461	AssertRC(rc);
1462	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1463	}
1464	}
1465
1466
1467	/**
1468	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1469	*
1470	* @param pVmxTransient The VMX-transient structure.
1471	*/
1472	DECLINLINE(void) hmR0VmxReadGuestPhysicalAddrVmcs(PVMXTRANSIENT pVmxTransient)
1473	{
1474	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1475	{
1476	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1477	AssertRC(rc);
1478	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1479	}
1480	}
1481
1482	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1483	/**
1484	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1485	* structure.
1486	*
1487	* @param pVmxTransient The VMX-transient structure.
1488	*/
1489	DECLINLINE(void) hmR0VmxReadGuestPendingDbgXctps(PVMXTRANSIENT pVmxTransient)
1490	{
1491	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1492	{
1493	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1494	AssertRC(rc);
1495	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1496	}
1497	}
1498	#endif
1499
1500	/**
1501	* Reads the IDT-vectoring information field from the VMCS into the VMX
1502	* transient structure.
1503	*
1504	* @param pVmxTransient The VMX-transient structure.
1505	*
1506	* @remarks No-long-jump zone!!!
1507	*/
1508	DECLINLINE(void) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1509	{
1510	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1511	{
1512	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1513	AssertRC(rc);
1514	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1515	}
1516	}
1517
1518
1519	/**
1520	* Reads the IDT-vectoring error code from the VMCS into the VMX
1521	* transient structure.
1522	*
1523	* @param pVmxTransient The VMX-transient structure.
1524	*/
1525	DECLINLINE(void) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1526	{
1527	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1528	{
1529	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1530	AssertRC(rc);
1531	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1532	}
1533	}
1534
1535	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1536	/**
1537	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1538	*
1539	* @param pVmxTransient The VMX-transient structure.
1540	*/
1541	static void hmR0VmxReadAllRoFieldsVmcs(PVMXTRANSIENT pVmxTransient)
1542	{
1543	int rc = VMXReadVmcsNw(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1544	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1545	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1546	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1547	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1548	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1549	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1550	rc \|= VMXReadVmcsNw(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1551	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1552	AssertRC(rc);
1553	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1554	\| HMVMX_READ_EXIT_INSTR_LEN
1555	\| HMVMX_READ_EXIT_INSTR_INFO
1556	\| HMVMX_READ_IDT_VECTORING_INFO
1557	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1558	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1559	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1560	\| HMVMX_READ_GUEST_LINEAR_ADDR
1561	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1562	}
1563	#endif
1564
1565	/**
1566	* Enters VMX root mode operation on the current CPU.
1567	*
1568	* @returns VBox status code.
1569	* @param pVM The cross context VM structure. Can be
1570	* NULL, after a resume.
1571	* @param HCPhysCpuPage Physical address of the VMXON region.
1572	* @param pvCpuPage Pointer to the VMXON region.
1573	*/
1574	static int hmR0VmxEnterRootMode(PVMCC pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1575	{
1576	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1577	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1578	Assert(pvCpuPage);
1579	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1580
1581	if (pVM)
1582	{
1583	/* Write the VMCS revision identifier to the VMXON region. */
1584	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1585	}
1586
1587	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1588	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1589
1590	/* Enable the VMX bit in CR4 if necessary. */
1591	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1592
1593	/* Enter VMX root mode. */
1594	int rc = VMXEnable(HCPhysCpuPage);
1595	if (RT_FAILURE(rc))
1596	{
1597	if (!(uOldCr4 & X86_CR4_VMXE))
1598	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1599
1600	if (pVM)
1601	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1602	}
1603
1604	/* Restore interrupts. */
1605	ASMSetFlags(fEFlags);
1606	return rc;
1607	}
1608
1609
1610	/**
1611	* Exits VMX root mode operation on the current CPU.
1612	*
1613	* @returns VBox status code.
1614	*/
1615	static int hmR0VmxLeaveRootMode(void)
1616	{
1617	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1618
1619	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1620	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1621
1622	/* If we're for some reason not in VMX root mode, then don't leave it. */
1623	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1624
1625	int rc;
1626	if (uHostCr4 & X86_CR4_VMXE)
1627	{
1628	/* Exit VMX root mode and clear the VMX bit in CR4. */
1629	VMXDisable();
1630	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1631	rc = VINF_SUCCESS;
1632	}
1633	else
1634	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1635
1636	/* Restore interrupts. */
1637	ASMSetFlags(fEFlags);
1638	return rc;
1639	}
1640
1641
1642	/**
1643	* Allocates and maps a physically contiguous page. The allocated page is
1644	* zero'd out (used by various VT-x structures).
1645	*
1646	* @returns IPRT status code.
1647	* @param pMemObj Pointer to the ring-0 memory object.
1648	* @param ppVirt Where to store the virtual address of the allocation.
1649	* @param pHCPhys Where to store the physical address of the allocation.
1650	*/
1651	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1652	{
1653	AssertPtr(pMemObj);
1654	AssertPtr(ppVirt);
1655	AssertPtr(pHCPhys);
1656	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1657	if (RT_FAILURE(rc))
1658	return rc;
1659	ppVirt = RTR0MemObjAddress(pMemObj);
1660	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1661	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1662	return VINF_SUCCESS;
1663	}
1664
1665
1666	/**
1667	* Frees and unmaps an allocated, physical page.
1668	*
1669	* @param pMemObj Pointer to the ring-0 memory object.
1670	* @param ppVirt Where to re-initialize the virtual address of allocation as
1671	* 0.
1672	* @param pHCPhys Where to re-initialize the physical address of the
1673	* allocation as 0.
1674	*/
1675	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1676	{
1677	AssertPtr(pMemObj);
1678	AssertPtr(ppVirt);
1679	AssertPtr(pHCPhys);
1680	/* NULL is valid, accepted and ignored by the free function below. */
1681	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1682	*pMemObj = NIL_RTR0MEMOBJ;
1683	*ppVirt = NULL;
1684	*pHCPhys = NIL_RTHCPHYS;
1685	}
1686
1687
1688	/**
1689	* Initializes a VMCS info. object.
1690	*
1691	* @param pVmcsInfo The VMCS info. object.
1692	*/
1693	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1694	{
1695	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1696
1697	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1698	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1699	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1700	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1701	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1702	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1703	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1704	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1705	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1706	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1707	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1708	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1709	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1710	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1711	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1712	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
1713	pVmcsInfo->idHostCpuExec = NIL_RTCPUID;
1714	}
1715
1716
1717	/**
1718	* Frees the VT-x structures for a VMCS info. object.
1719	*
1720	* @param pVM The cross context VM structure.
1721	* @param pVmcsInfo The VMCS info. object.
1722	*/
1723	static void hmR0VmxFreeVmcsInfo(PVMCC pVM, PVMXVMCSINFO pVmcsInfo)
1724	{
1725	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1726
1727	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1728	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1729	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1730	#endif
1731
1732	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1733	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1734
1735	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1736	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1737	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1738
1739	hmR0VmxInitVmcsInfo(pVmcsInfo);
1740	}
1741
1742
1743	/**
1744	* Allocates the VT-x structures for a VMCS info. object.
1745	*
1746	* @returns VBox status code.
1747	* @param pVCpu The cross context virtual CPU structure.
1748	* @param pVmcsInfo The VMCS info. object.
1749	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1750	*/
1751	static int hmR0VmxAllocVmcsInfo(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1752	{
1753	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1754
1755	/* Allocate the guest VM control structure (VMCS). */
1756	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1757	if (RT_SUCCESS(rc))
1758	{
1759	if (!fIsNstGstVmcs)
1760	{
1761	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1762	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1763	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1764	#endif
1765	if (RT_SUCCESS(rc))
1766	{
1767	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1768	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1769	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1770	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1771	}
1772	}
1773	else
1774	{
1775	/* We don't yet support exposing VMCS shadowing to the guest. */
1776	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1777	Assert(!pVmcsInfo->pvShadowVmcs);
1778
1779	/* Get the allocated virtual-APIC page from CPUM. */
1780	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1781	{
1782	/** @todo NSTVMX: Get rid of this. There is no need to allocate a separate HC
1783	* page for this. Use the one provided by the nested-guest directly. */
1784	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1785	&pVmcsInfo->HCPhysVirtApic);
1786	Assert(pVmcsInfo->pbVirtApic);
1787	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1788	}
1789	}
1790
1791	if (RT_SUCCESS(rc))
1792	{
1793	/*
1794	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1795	* transparent accesses of specific MSRs.
1796	*
1797	* If the condition for enabling MSR bitmaps changes here, don't forget to
1798	* update HMIsMsrBitmapActive().
1799	*
1800	* We don't share MSR bitmaps between the guest and nested-guest as we then
1801	* don't need to care about carefully restoring the guest MSR bitmap.
1802	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1803	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1804	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1805	* we do that later while merging VMCS.
1806	*/
1807	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1808	{
1809	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1810	if ( RT_SUCCESS(rc)
1811	&& !fIsNstGstVmcs)
1812	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1813	}
1814
1815	if (RT_SUCCESS(rc))
1816	{
1817	/*
1818	* Allocate the VM-entry MSR-load area for the guest MSRs.
1819	*
1820	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1821	* the guest and nested-guest.
1822	*/
1823	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1824	&pVmcsInfo->HCPhysGuestMsrLoad);
1825	if (RT_SUCCESS(rc))
1826	{
1827	/*
1828	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1829	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1830	*/
1831	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1832	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1833	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1834
1835	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1836	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1837	&pVmcsInfo->HCPhysHostMsrLoad);
1838	}
1839	}
1840	}
1841	}
1842
1843	return rc;
1844	}
1845
1846
1847	/**
1848	* Free all VT-x structures for the VM.
1849	*
1850	* @returns IPRT status code.
1851	* @param pVM The cross context VM structure.
1852	*/
1853	static void hmR0VmxStructsFree(PVMCC pVM)
1854	{
1855	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1856	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1857	#endif
1858	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1859
1860	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1861	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1862	{
1863	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1864	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1865	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1866	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1867	}
1868	#endif
1869
1870	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1871	{
1872	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1873	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1874	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1875	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1876	if (pVM->cpum.ro.GuestFeatures.fVmx)
1877	{
1878	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1879	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1880	}
1881	#endif
1882	}
1883	}
1884
1885
1886	/**
1887	* Allocate all VT-x structures for the VM.
1888	*
1889	* @returns IPRT status code.
1890	* @param pVM The cross context VM structure.
1891	*/
1892	static int hmR0VmxStructsAlloc(PVMCC pVM)
1893	{
1894	/*
1895	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1896	* The VMCS size cannot be more than 4096 bytes.
1897	*
1898	* See Intel spec. Appendix A.1 "Basic VMX Information".
1899	*/
1900	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1901	if (cbVmcs <= X86_PAGE_4K_SIZE)
1902	{ /* likely */ }
1903	else
1904	{
1905	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1906	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1907	}
1908
1909	/*
1910	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1911	*/
1912	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1913	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1914	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1915	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1916	#endif
1917
1918	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1919	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1920	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1921
1922	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1923	{
1924	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1925	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1926	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1927	}
1928
1929	/*
1930	* Allocate per-VM VT-x structures.
1931	*/
1932	int rc = VINF_SUCCESS;
1933	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1934	/* Allocate crash-dump magic scratch page. */
1935	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1936	if (RT_FAILURE(rc))
1937	{
1938	hmR0VmxStructsFree(pVM);
1939	return rc;
1940	}
1941	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1942	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1943	#endif
1944
1945	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1946	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1947	{
1948	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1949	&pVM->hm.s.vmx.HCPhysApicAccess);
1950	if (RT_FAILURE(rc))
1951	{
1952	hmR0VmxStructsFree(pVM);
1953	return rc;
1954	}
1955	}
1956
1957	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1958	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1959	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1960	{
1961	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1962	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1963	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1964	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1965	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1966	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1967	{
1968	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1969	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1970	if (RT_SUCCESS(rc))
1971	{
1972	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1973	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1974	}
1975	}
1976	else
1977	rc = VERR_NO_MEMORY;
1978
1979	if (RT_FAILURE(rc))
1980	{
1981	hmR0VmxStructsFree(pVM);
1982	return rc;
1983	}
1984	}
1985	#endif
1986
1987	/*
1988	* Initialize per-VCPU VT-x structures.
1989	*/
1990	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1991	{
1992	/* Allocate the guest VMCS structures. */
1993	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1994	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1995	if (RT_SUCCESS(rc))
1996	{
1997	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1998	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1999	if (pVM->cpum.ro.GuestFeatures.fVmx)
2000	{
2001	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2002	if (RT_SUCCESS(rc))
2003	{ /* likely */ }
2004	else
2005	break;
2006	}
2007	#endif
2008	}
2009	else
2010	break;
2011	}
2012
2013	if (RT_FAILURE(rc))
2014	{
2015	hmR0VmxStructsFree(pVM);
2016	return rc;
2017	}
2018
2019	return VINF_SUCCESS;
2020	}
2021
2022	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2023	/**
2024	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2025	*
2026	* @returns @c true if the MSR is intercepted, @c false otherwise.
2027	* @param pvMsrBitmap The MSR bitmap.
2028	* @param offMsr The MSR byte offset.
2029	* @param iBit The bit offset from the byte offset.
2030	*/
2031	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2032	{
2033	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2034	Assert(pbMsrBitmap);
2035	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2036	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2037	}
2038	#endif
2039
2040	/**
2041	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2042	*
2043	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2044	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2045	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2046	* the read/write access of this MSR.
2047	*
2048	* @param pVCpu The cross context virtual CPU structure.
2049	* @param pVmcsInfo The VMCS info. object.
2050	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2051	* @param idMsr The MSR value.
2052	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2053	* include both a read -and- a write permission!
2054	*
2055	* @sa CPUMGetVmxMsrPermission.
2056	* @remarks Can be called with interrupts disabled.
2057	*/
2058	static void hmR0VmxSetMsrPermission(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2059	{
2060	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2061	Assert(pbMsrBitmap);
2062	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2063
2064	/*
2065	* MSR-bitmap Layout:
2066	* Byte index MSR range Interpreted as
2067	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2068	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2069	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2070	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2071	*
2072	* A bit corresponding to an MSR within the above range causes a VM-exit
2073	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2074	* the MSR range, it always cause a VM-exit.
2075	*
2076	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2077	*/
2078	uint16_t const offBitmapRead = 0;
2079	uint16_t const offBitmapWrite = 0x800;
2080	uint16_t offMsr;
2081	int32_t iBit;
2082	if (idMsr <= UINT32_C(0x00001fff))
2083	{
2084	offMsr = 0;
2085	iBit = idMsr;
2086	}
2087	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2088	{
2089	offMsr = 0x400;
2090	iBit = idMsr - UINT32_C(0xc0000000);
2091	}
2092	else
2093	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2094
2095	/*
2096	* Set the MSR read permission.
2097	*/
2098	uint16_t const offMsrRead = offBitmapRead + offMsr;
2099	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2100	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2101	{
2102	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2103	bool const fClear = !fIsNstGstVmcs ? true
2104	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2105	#else
2106	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2107	bool const fClear = true;
2108	#endif
2109	if (fClear)
2110	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2111	}
2112	else
2113	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2114
2115	/*
2116	* Set the MSR write permission.
2117	*/
2118	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2119	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2120	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2121	{
2122	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2123	bool const fClear = !fIsNstGstVmcs ? true
2124	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2125	#else
2126	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2127	bool const fClear = true;
2128	#endif
2129	if (fClear)
2130	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2131	}
2132	else
2133	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2134	}
2135
2136
2137	/**
2138	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2139	* area.
2140	*
2141	* @returns VBox status code.
2142	* @param pVCpu The cross context virtual CPU structure.
2143	* @param pVmcsInfo The VMCS info. object.
2144	* @param cMsrs The number of MSRs.
2145	*/
2146	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2147	{
2148	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2149	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2150	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2151	{
2152	/* Commit the MSR counts to the VMCS and update the cache. */
2153	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2154	{
2155	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2156	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRC(rc);
2157	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRC(rc);
2158	pVmcsInfo->cEntryMsrLoad = cMsrs;
2159	pVmcsInfo->cExitMsrStore = cMsrs;
2160	pVmcsInfo->cExitMsrLoad = cMsrs;
2161	}
2162	return VINF_SUCCESS;
2163	}
2164
2165	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2166	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2167	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2168	}
2169
2170
2171	/**
2172	* Adds a new (or updates the value of an existing) guest/host MSR
2173	* pair to be swapped during the world-switch as part of the
2174	* auto-load/store MSR area in the VMCS.
2175	*
2176	* @returns VBox status code.
2177	* @param pVCpu The cross context virtual CPU structure.
2178	* @param pVmxTransient The VMX-transient structure.
2179	* @param idMsr The MSR.
2180	* @param uGuestMsrValue Value of the guest MSR.
2181	* @param fSetReadWrite Whether to set the guest read/write access of this
2182	* MSR (thus not causing a VM-exit).
2183	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2184	* necessary.
2185	*/
2186	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2187	bool fSetReadWrite, bool fUpdateHostMsr)
2188	{
2189	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2190	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2191	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2192	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2193	uint32_t i;
2194
2195	/* Paranoia. */
2196	Assert(pGuestMsrLoad);
2197
2198	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2199
2200	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2201	for (i = 0; i < cMsrs; i++)
2202	{
2203	if (pGuestMsrLoad[i].u32Msr == idMsr)
2204	break;
2205	}
2206
2207	bool fAdded = false;
2208	if (i == cMsrs)
2209	{
2210	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2211	++cMsrs;
2212	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2213	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2214
2215	/* Set the guest to read/write this MSR without causing VM-exits. */
2216	if ( fSetReadWrite
2217	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2218	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2219
2220	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2221	fAdded = true;
2222	}
2223
2224	/* Update the MSR value for the newly added or already existing MSR. */
2225	pGuestMsrLoad[i].u32Msr = idMsr;
2226	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2227
2228	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2229	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2230	{
2231	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2232	pGuestMsrStore[i].u32Msr = idMsr;
2233	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2234	}
2235
2236	/* Update the corresponding slot in the host MSR area. */
2237	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2238	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2239	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2240	pHostMsr[i].u32Msr = idMsr;
2241
2242	/*
2243	* Only if the caller requests to update the host MSR value AND we've newly added the
2244	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2245	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2246	*
2247	* We do this for performance reasons since reading MSRs may be quite expensive.
2248	*/
2249	if (fAdded)
2250	{
2251	if (fUpdateHostMsr)
2252	{
2253	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2254	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2255	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2256	}
2257	else
2258	{
2259	/* Someone else can do the work. */
2260	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2261	}
2262	}
2263	return VINF_SUCCESS;
2264	}
2265
2266
2267	/**
2268	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2269	* auto-load/store MSR area in the VMCS.
2270	*
2271	* @returns VBox status code.
2272	* @param pVCpu The cross context virtual CPU structure.
2273	* @param pVmxTransient The VMX-transient structure.
2274	* @param idMsr The MSR.
2275	*/
2276	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2277	{
2278	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2279	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2280	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2281	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2282
2283	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2284
2285	for (uint32_t i = 0; i < cMsrs; i++)
2286	{
2287	/* Find the MSR. */
2288	if (pGuestMsrLoad[i].u32Msr == idMsr)
2289	{
2290	/*
2291	* If it's the last MSR, we only need to reduce the MSR count.
2292	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2293	*/
2294	if (i < cMsrs - 1)
2295	{
2296	/* Remove it from the VM-entry MSR-load area. */
2297	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2298	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2299
2300	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2301	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2302	{
2303	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2304	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2305	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2306	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2307	}
2308
2309	/* Remove it from the VM-exit MSR-load area. */
2310	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2311	Assert(pHostMsr[i].u32Msr == idMsr);
2312	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2313	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2314	}
2315
2316	/* Reduce the count to reflect the removed MSR and bail. */
2317	--cMsrs;
2318	break;
2319	}
2320	}
2321
2322	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2323	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2324	{
2325	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2326	AssertRCReturn(rc, rc);
2327
2328	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2329	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2330	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2331
2332	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2333	return VINF_SUCCESS;
2334	}
2335
2336	return VERR_NOT_FOUND;
2337	}
2338
2339
2340	/**
2341	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2342	*
2343	* @returns @c true if found, @c false otherwise.
2344	* @param pVmcsInfo The VMCS info. object.
2345	* @param idMsr The MSR to find.
2346	*/
2347	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2348	{
2349	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2350	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2351	Assert(pMsrs);
2352	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2353	for (uint32_t i = 0; i < cMsrs; i++)
2354	{
2355	if (pMsrs[i].u32Msr == idMsr)
2356	return true;
2357	}
2358	return false;
2359	}
2360
2361
2362	/**
2363	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2364	*
2365	* @param pVCpu The cross context virtual CPU structure.
2366	* @param pVmcsInfo The VMCS info. object.
2367	*
2368	* @remarks No-long-jump zone!!!
2369	*/
2370	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2371	{
2372	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2373
2374	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2375	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2376	Assert(pHostMsrLoad);
2377	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2378	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2379	for (uint32_t i = 0; i < cMsrs; i++)
2380	{
2381	/*
2382	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2383	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2384	*/
2385	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2386	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2387	else
2388	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2389	}
2390	}
2391
2392
2393	/**
2394	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2395	* perform lazy restoration of the host MSRs while leaving VT-x.
2396	*
2397	* @param pVCpu The cross context virtual CPU structure.
2398	*
2399	* @remarks No-long-jump zone!!!
2400	*/
2401	static void hmR0VmxLazySaveHostMsrs(PVMCPUCC pVCpu)
2402	{
2403	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2404
2405	/*
2406	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2407	*/
2408	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2409	{
2410	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2411	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2412	{
2413	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2414	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2415	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2416	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2417	}
2418	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2419	}
2420	}
2421
2422
2423	/**
2424	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2425	* lazily while leaving VT-x.
2426	*
2427	* @returns true if it does, false otherwise.
2428	* @param pVCpu The cross context virtual CPU structure.
2429	* @param idMsr The MSR to check.
2430	*/
2431	static bool hmR0VmxIsLazyGuestMsr(PCVMCPUCC pVCpu, uint32_t idMsr)
2432	{
2433	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2434	{
2435	switch (idMsr)
2436	{
2437	case MSR_K8_LSTAR:
2438	case MSR_K6_STAR:
2439	case MSR_K8_SF_MASK:
2440	case MSR_K8_KERNEL_GS_BASE:
2441	return true;
2442	}
2443	}
2444	return false;
2445	}
2446
2447
2448	/**
2449	* Loads a set of guests MSRs to allow read/passthru to the guest.
2450	*
2451	* The name of this function is slightly confusing. This function does NOT
2452	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2453	* common prefix for functions dealing with "lazy restoration" of the shared
2454	* MSRs.
2455	*
2456	* @param pVCpu The cross context virtual CPU structure.
2457	*
2458	* @remarks No-long-jump zone!!!
2459	*/
2460	static void hmR0VmxLazyLoadGuestMsrs(PVMCPUCC pVCpu)
2461	{
2462	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2463	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2464
2465	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2466	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2467	{
2468	/*
2469	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2470	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2471	* we can skip a few MSR writes.
2472	*
2473	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2474	* guest MSR values in the guest-CPU context might be different to what's currently
2475	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2476	* CPU, see @bugref{8728}.
2477	*/
2478	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2479	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2480	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2481	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2482	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2483	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2484	{
2485	#ifdef VBOX_STRICT
2486	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2487	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2488	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2489	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2490	#endif
2491	}
2492	else
2493	{
2494	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2495	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2496	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2497	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2498	}
2499	}
2500	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2501	}
2502
2503
2504	/**
2505	* Performs lazy restoration of the set of host MSRs if they were previously
2506	* loaded with guest MSR values.
2507	*
2508	* @param pVCpu The cross context virtual CPU structure.
2509	*
2510	* @remarks No-long-jump zone!!!
2511	* @remarks The guest MSRs should have been saved back into the guest-CPU
2512	* context by hmR0VmxImportGuestState()!!!
2513	*/
2514	static void hmR0VmxLazyRestoreHostMsrs(PVMCPUCC pVCpu)
2515	{
2516	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2517	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2518
2519	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2520	{
2521	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2522	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2523	{
2524	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2525	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2526	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2527	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2528	}
2529	}
2530	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2531	}
2532
2533
2534	/**
2535	* Verifies that our cached values of the VMCS fields are all consistent with
2536	* what's actually present in the VMCS.
2537	*
2538	* @returns VBox status code.
2539	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2540	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2541	* VMCS content. HMCPU error-field is
2542	* updated, see VMX_VCI_XXX.
2543	* @param pVCpu The cross context virtual CPU structure.
2544	* @param pVmcsInfo The VMCS info. object.
2545	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2546	*/
2547	static int hmR0VmxCheckVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2548	{
2549	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2550
2551	uint32_t u32Val;
2552	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2553	AssertRC(rc);
2554	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2555	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2556	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2557	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2558
2559	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2560	AssertRC(rc);
2561	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2562	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2563	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2564	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2565
2566	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2567	AssertRC(rc);
2568	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2569	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2570	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2571	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2572
2573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2574	AssertRC(rc);
2575	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2576	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2577	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2578	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2579
2580	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2581	{
2582	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2583	AssertRC(rc);
2584	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2585	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2586	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2587	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2588	}
2589
2590	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2591	AssertRC(rc);
2592	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2593	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2594	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2595	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2596
2597	uint64_t u64Val;
2598	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2599	AssertRC(rc);
2600	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2601	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2602	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2603	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2604
2605	NOREF(pcszVmcs);
2606	return VINF_SUCCESS;
2607	}
2608
2609
2610	#ifdef VBOX_STRICT
2611	/**
2612	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2613	*
2614	* @param pVCpu The cross context virtual CPU structure.
2615	* @param pVmcsInfo The VMCS info. object.
2616	*/
2617	static void hmR0VmxCheckHostEferMsr(PCVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2618	{
2619	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2620
2621	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2622	{
2623	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2624	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2625	uint64_t uVmcsEferMsrVmcs;
2626	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2627	AssertRC(rc);
2628
2629	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2630	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2631	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2632	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2633	}
2634	}
2635
2636
2637	/**
2638	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2639	* VMCS are correct.
2640	*
2641	* @param pVCpu The cross context virtual CPU structure.
2642	* @param pVmcsInfo The VMCS info. object.
2643	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2644	*/
2645	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2646	{
2647	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2648
2649	/* Read the various MSR-area counts from the VMCS. */
2650	uint32_t cEntryLoadMsrs;
2651	uint32_t cExitStoreMsrs;
2652	uint32_t cExitLoadMsrs;
2653	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2654	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2655	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2656
2657	/* Verify all the MSR counts are the same. */
2658	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2659	Assert(cExitStoreMsrs == cExitLoadMsrs);
2660	uint32_t const cMsrs = cExitLoadMsrs;
2661
2662	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2663	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2664
2665	/* Verify the MSR counts are within the allocated page size. */
2666	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2667
2668	/* Verify the relevant contents of the MSR areas match. */
2669	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2670	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2671	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2672	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2673	for (uint32_t i = 0; i < cMsrs; i++)
2674	{
2675	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2676	if (fSeparateExitMsrStorePage)
2677	{
2678	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2679	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2680	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2681	}
2682
2683	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2684	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2685	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2686
2687	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2688	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2689	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2690	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2691
2692	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2693	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2694	if (fIsEferMsr)
2695	{
2696	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2697	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2698	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2699	}
2700
2701	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2702	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2703	{
2704	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2705	if (fIsEferMsr)
2706	{
2707	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2708	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2709	}
2710	else
2711	{
2712	if (!fIsNstGstVmcs)
2713	{
2714	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2715	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2716	}
2717	else
2718	{
2719	/*
2720	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2721	* execute a nested-guest with MSR passthrough.
2722	*
2723	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2724	* allow passthrough too.
2725	*/
2726	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2727	Assert(pvMsrBitmapNstGst);
2728	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2729	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2730	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2731	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2732	}
2733	}
2734	}
2735
2736	/* Move to the next MSR. */
2737	pHostMsrLoad++;
2738	pGuestMsrLoad++;
2739	pGuestMsrStore++;
2740	}
2741	}
2742	#endif /* VBOX_STRICT */
2743
2744
2745	/**
2746	* Flushes the TLB using EPT.
2747	*
2748	* @returns VBox status code.
2749	* @param pVCpu The cross context virtual CPU structure of the calling
2750	* EMT. Can be NULL depending on @a enmTlbFlush.
2751	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2752	* enmTlbFlush.
2753	* @param enmTlbFlush Type of flush.
2754	*
2755	* @remarks Caller is responsible for making sure this function is called only
2756	* when NestedPaging is supported and providing @a enmTlbFlush that is
2757	* supported by the CPU.
2758	* @remarks Can be called with interrupts disabled.
2759	*/
2760	static void hmR0VmxFlushEpt(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2761	{
2762	uint64_t au64Descriptor[2];
2763	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2764	au64Descriptor[0] = 0;
2765	else
2766	{
2767	Assert(pVCpu);
2768	Assert(pVmcsInfo);
2769	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2770	}
2771	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2772
2773	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2774	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2775
2776	if ( RT_SUCCESS(rc)
2777	&& pVCpu)
2778	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2779	}
2780
2781
2782	/**
2783	* Flushes the TLB using VPID.
2784	*
2785	* @returns VBox status code.
2786	* @param pVCpu The cross context virtual CPU structure of the calling
2787	* EMT. Can be NULL depending on @a enmTlbFlush.
2788	* @param enmTlbFlush Type of flush.
2789	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2790	* on @a enmTlbFlush).
2791	*
2792	* @remarks Can be called with interrupts disabled.
2793	*/
2794	static void hmR0VmxFlushVpid(PVMCPUCC pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2795	{
2796	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2797
2798	uint64_t au64Descriptor[2];
2799	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2800	{
2801	au64Descriptor[0] = 0;
2802	au64Descriptor[1] = 0;
2803	}
2804	else
2805	{
2806	AssertPtr(pVCpu);
2807	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2808	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2809	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2810	au64Descriptor[1] = GCPtr;
2811	}
2812
2813	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2814	AssertMsg(rc == VINF_SUCCESS,
2815	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2816
2817	if ( RT_SUCCESS(rc)
2818	&& pVCpu)
2819	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2820	NOREF(rc);
2821	}
2822
2823
2824	/**
2825	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2826	* otherwise there is nothing really to invalidate.
2827	*
2828	* @returns VBox status code.
2829	* @param pVCpu The cross context virtual CPU structure.
2830	* @param GCVirt Guest virtual address of the page to invalidate.
2831	*/
2832	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
2833	{
2834	AssertPtr(pVCpu);
2835	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2836
2837	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2838	{
2839	/*
2840	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2841	* the EPT case. See @bugref{6043} and @bugref{6177}.
2842	*
2843	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2844	* as this function maybe called in a loop with individual addresses.
2845	*/
2846	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2847	if (pVM->hm.s.vmx.fVpid)
2848	{
2849	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2850	if (fVpidFlush)
2851	{
2852	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2853	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2854	}
2855	else
2856	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2857	}
2858	else if (pVM->hm.s.fNestedPaging)
2859	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2860	}
2861
2862	return VINF_SUCCESS;
2863	}
2864
2865
2866	/**
2867	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2868	* case where neither EPT nor VPID is supported by the CPU.
2869	*
2870	* @param pHostCpu The HM physical-CPU structure.
2871	* @param pVCpu The cross context virtual CPU structure.
2872	*
2873	* @remarks Called with interrupts disabled.
2874	*/
2875	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2876	{
2877	AssertPtr(pVCpu);
2878	AssertPtr(pHostCpu);
2879
2880	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2881
2882	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2883	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2884	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2885	pVCpu->hm.s.fForceTLBFlush = false;
2886	return;
2887	}
2888
2889
2890	/**
2891	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2892	*
2893	* @param pHostCpu The HM physical-CPU structure.
2894	* @param pVCpu The cross context virtual CPU structure.
2895	* @param pVmcsInfo The VMCS info. object.
2896	*
2897	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2898	* nomenclature. The reason is, to avoid confusion in compare statements
2899	* since the host-CPU copies are named "ASID".
2900	*
2901	* @remarks Called with interrupts disabled.
2902	*/
2903	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
2904	{
2905	#ifdef VBOX_WITH_STATISTICS
2906	bool fTlbFlushed = false;
2907	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2908	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2909	if (!fTlbFlushed) \
2910	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2911	} while (0)
2912	#else
2913	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2914	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2915	#endif
2916
2917	AssertPtr(pVCpu);
2918	AssertPtr(pHostCpu);
2919	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2920
2921	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2922	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2923	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2924	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2925
2926	/*
2927	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2928	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2929	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2930	* cannot reuse the current ASID anymore.
2931	*/
2932	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2933	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2934	{
2935	++pHostCpu->uCurrentAsid;
2936	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2937	{
2938	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2939	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2940	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2941	}
2942
2943	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2944	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2945	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2946
2947	/*
2948	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2949	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2950	*/
2951	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2952	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2953	HMVMX_SET_TAGGED_TLB_FLUSHED();
2954	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2955	}
2956	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2957	{
2958	/*
2959	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2960	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2961	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2962	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2963	* mappings, see @bugref{6568}.
2964	*
2965	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2966	*/
2967	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2968	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2969	HMVMX_SET_TAGGED_TLB_FLUSHED();
2970	}
2971	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
2972	{
2973	/*
2974	* The nested-guest specifies its own guest-physical address to use as the APIC-access
2975	* address which requires flushing the TLB of EPT cached structures.
2976	*
2977	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
2978	*/
2979	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2980	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
2981	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
2982	HMVMX_SET_TAGGED_TLB_FLUSHED();
2983	}
2984
2985
2986	pVCpu->hm.s.fForceTLBFlush = false;
2987	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2988
2989	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2990	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2991	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2992	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2993	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2994	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2995	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2996	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2997	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2998
2999	/* Update VMCS with the VPID. */
3000	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3001	AssertRC(rc);
3002
3003	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3004	}
3005
3006
3007	/**
3008	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3009	*
3010	* @param pHostCpu The HM physical-CPU structure.
3011	* @param pVCpu The cross context virtual CPU structure.
3012	* @param pVmcsInfo The VMCS info. object.
3013	*
3014	* @remarks Called with interrupts disabled.
3015	*/
3016	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3017	{
3018	AssertPtr(pVCpu);
3019	AssertPtr(pHostCpu);
3020	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3021	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3022	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3023
3024	/*
3025	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3026	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3027	*/
3028	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3029	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3030	{
3031	pVCpu->hm.s.fForceTLBFlush = true;
3032	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3033	}
3034
3035	/* Check for explicit TLB flushes. */
3036	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3037	{
3038	pVCpu->hm.s.fForceTLBFlush = true;
3039	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3040	}
3041
3042	/* Check for TLB flushes while switching to/from a nested-guest. */
3043	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3044	{
3045	pVCpu->hm.s.fForceTLBFlush = true;
3046	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3047	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3048	}
3049
3050	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3051	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3052
3053	if (pVCpu->hm.s.fForceTLBFlush)
3054	{
3055	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3056	pVCpu->hm.s.fForceTLBFlush = false;
3057	}
3058	}
3059
3060
3061	/**
3062	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3063	*
3064	* @param pHostCpu The HM physical-CPU structure.
3065	* @param pVCpu The cross context virtual CPU structure.
3066	*
3067	* @remarks Called with interrupts disabled.
3068	*/
3069	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
3070	{
3071	AssertPtr(pVCpu);
3072	AssertPtr(pHostCpu);
3073	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3074	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3075	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3076
3077	/*
3078	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3079	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3080	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3081	* cannot reuse the current ASID anymore.
3082	*/
3083	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3084	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3085	{
3086	pVCpu->hm.s.fForceTLBFlush = true;
3087	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3088	}
3089
3090	/* Check for explicit TLB flushes. */
3091	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3092	{
3093	/*
3094	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3095	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3096	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3097	* include fExplicitFlush's too) - an obscure corner case.
3098	*/
3099	pVCpu->hm.s.fForceTLBFlush = true;
3100	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3101	}
3102
3103	/* Check for TLB flushes while switching to/from a nested-guest. */
3104	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3105	{
3106	pVCpu->hm.s.fForceTLBFlush = true;
3107	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3108	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3109	}
3110
3111	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3112	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3113	if (pVCpu->hm.s.fForceTLBFlush)
3114	{
3115	++pHostCpu->uCurrentAsid;
3116	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3117	{
3118	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3119	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3120	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3121	}
3122
3123	pVCpu->hm.s.fForceTLBFlush = false;
3124	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3125	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3126	if (pHostCpu->fFlushAsidBeforeUse)
3127	{
3128	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3129	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3130	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3131	{
3132	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3133	pHostCpu->fFlushAsidBeforeUse = false;
3134	}
3135	else
3136	{
3137	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3138	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3139	}
3140	}
3141	}
3142
3143	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3144	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3145	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3146	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3147	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3148	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3149	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3150
3151	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3152	AssertRC(rc);
3153	}
3154
3155
3156	/**
3157	* Flushes the guest TLB entry based on CPU capabilities.
3158	*
3159	* @param pHostCpu The HM physical-CPU structure.
3160	* @param pVCpu The cross context virtual CPU structure.
3161	* @param pVmcsInfo The VMCS info. object.
3162	*
3163	* @remarks Called with interrupts disabled.
3164	*/
3165	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3166	{
3167	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3168	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3169	#endif
3170	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3171	switch (pVM->hm.s.vmx.enmTlbFlushType)
3172	{
3173	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3174	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3175	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3176	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3177	default:
3178	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3179	break;
3180	}
3181	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3182	}
3183
3184
3185	/**
3186	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3187	* TLB entries from the host TLB before VM-entry.
3188	*
3189	* @returns VBox status code.
3190	* @param pVM The cross context VM structure.
3191	*/
3192	static int hmR0VmxSetupTaggedTlb(PVMCC pVM)
3193	{
3194	/*
3195	* Determine optimal flush type for nested paging.
3196	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3197	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3198	*/
3199	if (pVM->hm.s.fNestedPaging)
3200	{
3201	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3202	{
3203	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3204	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3205	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3206	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3207	else
3208	{
3209	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3210	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3211	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3212	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3213	}
3214
3215	/* Make sure the write-back cacheable memory type for EPT is supported. */
3216	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3217	{
3218	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3219	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3220	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3221	}
3222
3223	/* EPT requires a page-walk length of 4. */
3224	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3225	{
3226	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3227	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3228	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3229	}
3230	}
3231	else
3232	{
3233	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3234	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3235	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3236	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3237	}
3238	}
3239
3240	/*
3241	* Determine optimal flush type for VPID.
3242	*/
3243	if (pVM->hm.s.vmx.fVpid)
3244	{
3245	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3246	{
3247	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3248	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3249	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3250	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3251	else
3252	{
3253	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3254	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3255	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3256	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3257	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3258	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3259	pVM->hm.s.vmx.fVpid = false;
3260	}
3261	}
3262	else
3263	{
3264	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3265	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3266	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3267	pVM->hm.s.vmx.fVpid = false;
3268	}
3269	}
3270
3271	/*
3272	* Setup the handler for flushing tagged-TLBs.
3273	*/
3274	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3275	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3276	else if (pVM->hm.s.fNestedPaging)
3277	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3278	else if (pVM->hm.s.vmx.fVpid)
3279	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3280	else
3281	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3282	return VINF_SUCCESS;
3283	}
3284
3285
3286	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3287	/**
3288	* Sets up the shadow VMCS fields arrays.
3289	*
3290	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3291	* executing the guest.
3292	*
3293	* @returns VBox status code.
3294	* @param pVM The cross context VM structure.
3295	*/
3296	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVMCC pVM)
3297	{
3298	/*
3299	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3300	* when the host does not support it.
3301	*/
3302	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3303	if ( !fGstVmwriteAll
3304	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3305	{ /* likely. */ }
3306	else
3307	{
3308	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3309	VMCC_GET_CPU_0(pVM)->hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3310	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3311	}
3312
3313	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3314	uint32_t cRwFields = 0;
3315	uint32_t cRoFields = 0;
3316	for (uint32_t i = 0; i < cVmcsFields; i++)
3317	{
3318	VMXVMCSFIELD VmcsField;
3319	VmcsField.u = g_aVmcsFields[i];
3320
3321	/*
3322	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3323	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3324	* in the shadow VMCS fields array as they would be redundant.
3325	*
3326	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3327	* we must not include it in the shadow VMCS fields array. Guests attempting to
3328	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3329	* the required behavior.
3330	*/
3331	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3332	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3333	{
3334	/*
3335	* Read-only fields are placed in a separate array so that while syncing shadow
3336	* VMCS fields later (which is more performance critical) we can avoid branches.
3337	*
3338	* However, if the guest can write to all fields (including read-only fields),
3339	* we treat it a as read/write field. Otherwise, writing to these fields would
3340	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3341	*/
3342	if ( fGstVmwriteAll
3343	\|\| !VMXIsVmcsFieldReadOnly(VmcsField.u))
3344	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3345	else
3346	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3347	}
3348	}
3349
3350	/* Update the counts. */
3351	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3352	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3353	return VINF_SUCCESS;
3354	}
3355
3356
3357	/**
3358	* Sets up the VMREAD and VMWRITE bitmaps.
3359	*
3360	* @param pVM The cross context VM structure.
3361	*/
3362	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVMCC pVM)
3363	{
3364	/*
3365	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3366	*/
3367	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3368	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3369	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3370	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3371	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3372
3373	/*
3374	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3375	* VMREAD and VMWRITE bitmaps.
3376	*/
3377	{
3378	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3379	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3380	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3381	{
3382	uint32_t const uVmcsField = paShadowVmcsFields[i];
3383	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3384	Assert(uVmcsField >> 3 < cbBitmap);
3385	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3386	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3387	}
3388	}
3389
3390	/*
3391	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3392	* if the host supports VMWRITE to all supported VMCS fields.
3393	*/
3394	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3395	{
3396	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3397	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3398	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3399	{
3400	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3401	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3402	Assert(uVmcsField >> 3 < cbBitmap);
3403	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3404	}
3405	}
3406	}
3407	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3408
3409
3410	/**
3411	* Sets up the virtual-APIC page address for the VMCS.
3412	*
3413	* @param pVmcsInfo The VMCS info. object.
3414	*/
3415	DECLINLINE(void) hmR0VmxSetupVmcsVirtApicAddr(PCVMXVMCSINFO pVmcsInfo)
3416	{
3417	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3418	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3419	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3420	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3421	AssertRC(rc);
3422	}
3423
3424
3425	/**
3426	* Sets up the MSR-bitmap address for the VMCS.
3427	*
3428	* @param pVmcsInfo The VMCS info. object.
3429	*/
3430	DECLINLINE(void) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
3431	{
3432	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3433	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3434	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3435	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3436	AssertRC(rc);
3437	}
3438
3439
3440	/**
3441	* Sets up the APIC-access page address for the VMCS.
3442	*
3443	* @param pVCpu The cross context virtual CPU structure.
3444	*/
3445	DECLINLINE(void) hmR0VmxSetupVmcsApicAccessAddr(PVMCPUCC pVCpu)
3446	{
3447	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3448	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3449	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3450	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3451	AssertRC(rc);
3452	}
3453
3454
3455	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3456	/**
3457	* Sets up the VMREAD bitmap address for the VMCS.
3458	*
3459	* @param pVCpu The cross context virtual CPU structure.
3460	*/
3461	DECLINLINE(void) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPUCC pVCpu)
3462	{
3463	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3464	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3465	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3466	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3467	AssertRC(rc);
3468	}
3469
3470
3471	/**
3472	* Sets up the VMWRITE bitmap address for the VMCS.
3473	*
3474	* @param pVCpu The cross context virtual CPU structure.
3475	*/
3476	DECLINLINE(void) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPUCC pVCpu)
3477	{
3478	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3479	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3480	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3481	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3482	AssertRC(rc);
3483	}
3484	#endif
3485
3486
3487	/**
3488	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3489	* in the VMCS.
3490	*
3491	* @returns VBox status code.
3492	* @param pVmcsInfo The VMCS info. object.
3493	*/
3494	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
3495	{
3496	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3497	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3498	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3499
3500	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3501	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3502	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3503
3504	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3505	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3506	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3507
3508	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad); AssertRC(rc);
3509	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore); AssertRC(rc);
3510	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad); AssertRC(rc);
3511	return VINF_SUCCESS;
3512	}
3513
3514
3515	/**
3516	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3517	*
3518	* @param pVCpu The cross context virtual CPU structure.
3519	* @param pVmcsInfo The VMCS info. object.
3520	*/
3521	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3522	{
3523	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3524
3525	/*
3526	* The guest can access the following MSRs (read, write) without causing
3527	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3528	*/
3529	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3530	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3531	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3532	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3533	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3534	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3535
3536	/*
3537	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3538	* associated with then. We never need to intercept access (writes need to be
3539	* executed without causing a VM-exit, reads will #GP fault anyway).
3540	*
3541	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3542	* read/write them. We swap the the guest/host MSR value using the
3543	* auto-load/store MSR area.
3544	*/
3545	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3546	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3547	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3548	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3549	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3550	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3551
3552	/*
3553	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3554	* required for 64-bit guests.
3555	*/
3556	if (pVM->hm.s.fAllow64BitGuests)
3557	{
3558	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3559	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3560	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3561	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3562	}
3563
3564	/*
3565	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3566	*/
3567	#ifdef VBOX_STRICT
3568	Assert(pVmcsInfo->pvMsrBitmap);
3569	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3570	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3571	#endif
3572	}
3573
3574
3575	/**
3576	* Sets up pin-based VM-execution controls in the VMCS.
3577	*
3578	* @returns VBox status code.
3579	* @param pVCpu The cross context virtual CPU structure.
3580	* @param pVmcsInfo The VMCS info. object.
3581	*/
3582	static int hmR0VmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3583	{
3584	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3585	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3586	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3587
3588	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3589	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3590
3591	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3592	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3593
3594	/* Enable the VMX-preemption timer. */
3595	if (pVM->hm.s.vmx.fUsePreemptTimer)
3596	{
3597	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3598	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3599	}
3600
3601	#if 0
3602	/* Enable posted-interrupt processing. */
3603	if (pVM->hm.s.fPostedIntrs)
3604	{
3605	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3606	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3607	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3608	}
3609	#endif
3610
3611	if ((fVal & fZap) != fVal)
3612	{
3613	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3614	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3615	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3616	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3617	}
3618
3619	/* Commit it to the VMCS and update our cache. */
3620	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3621	AssertRC(rc);
3622	pVmcsInfo->u32PinCtls = fVal;
3623
3624	return VINF_SUCCESS;
3625	}
3626
3627
3628	/**
3629	* Sets up secondary processor-based VM-execution controls in the VMCS.
3630	*
3631	* @returns VBox status code.
3632	* @param pVCpu The cross context virtual CPU structure.
3633	* @param pVmcsInfo The VMCS info. object.
3634	*/
3635	static int hmR0VmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3636	{
3637	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3638	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3639	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3640
3641	/* WBINVD causes a VM-exit. */
3642	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3643	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3644
3645	/* Enable EPT (aka nested-paging). */
3646	if (pVM->hm.s.fNestedPaging)
3647	fVal \|= VMX_PROC_CTLS2_EPT;
3648
3649	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3650	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3651	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3652	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3653	fVal \|= VMX_PROC_CTLS2_INVPCID;
3654
3655	/* Enable VPID. */
3656	if (pVM->hm.s.vmx.fVpid)
3657	fVal \|= VMX_PROC_CTLS2_VPID;
3658
3659	/* Enable unrestricted guest execution. */
3660	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3661	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3662
3663	#if 0
3664	if (pVM->hm.s.fVirtApicRegs)
3665	{
3666	/* Enable APIC-register virtualization. */
3667	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3668	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3669
3670	/* Enable virtual-interrupt delivery. */
3671	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3672	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3673	}
3674	#endif
3675
3676	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3677	where the TPR shadow resides. */
3678	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3679	* done dynamically. */
3680	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3681	{
3682	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3683	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3684	}
3685
3686	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3687	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3688	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3689	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3690	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3691
3692	/* Enable Pause-Loop exiting. */
3693	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3694	&& pVM->hm.s.vmx.cPleGapTicks
3695	&& pVM->hm.s.vmx.cPleWindowTicks)
3696	{
3697	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3698
3699	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
3700	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
3701	}
3702
3703	if ((fVal & fZap) != fVal)
3704	{
3705	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3706	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3707	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3708	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3709	}
3710
3711	/* Commit it to the VMCS and update our cache. */
3712	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3713	AssertRC(rc);
3714	pVmcsInfo->u32ProcCtls2 = fVal;
3715
3716	return VINF_SUCCESS;
3717	}
3718
3719
3720	/**
3721	* Sets up processor-based VM-execution controls in the VMCS.
3722	*
3723	* @returns VBox status code.
3724	* @param pVCpu The cross context virtual CPU structure.
3725	* @param pVmcsInfo The VMCS info. object.
3726	*/
3727	static int hmR0VmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3728	{
3729	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3730
3731	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3732	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3733
3734	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3735	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3736	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3737	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3738	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3739	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3740	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3741
3742	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3743	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3744	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3745	{
3746	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3747	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3748	}
3749
3750	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3751	if (!pVM->hm.s.fNestedPaging)
3752	{
3753	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3754	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3755	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3756	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3757	}
3758
3759	/* Use TPR shadowing if supported by the CPU. */
3760	if ( PDMHasApic(pVM)
3761	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3762	{
3763	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3764	/* CR8 writes cause a VM-exit based on TPR threshold. */
3765	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3766	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3767	hmR0VmxSetupVmcsVirtApicAddr(pVmcsInfo);
3768	}
3769	else
3770	{
3771	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3772	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3773	if (pVM->hm.s.fAllow64BitGuests)
3774	{
3775	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3776	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3777	}
3778	}
3779
3780	/* Use MSR-bitmaps if supported by the CPU. */
3781	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3782	{
3783	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3784	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3785	}
3786
3787	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3788	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3789	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3790
3791	if ((fVal & fZap) != fVal)
3792	{
3793	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3794	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3795	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3796	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3797	}
3798
3799	/* Commit it to the VMCS and update our cache. */
3800	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3801	AssertRC(rc);
3802	pVmcsInfo->u32ProcCtls = fVal;
3803
3804	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3805	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3806	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3807
3808	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3809	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3810	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3811
3812	/* Sanity check, should not really happen. */
3813	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3814	{ /* likely */ }
3815	else
3816	{
3817	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3818	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3819	}
3820
3821	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3822	return VINF_SUCCESS;
3823	}
3824
3825
3826	/**
3827	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3828	* Processor-based VM-execution) control fields in the VMCS.
3829	*
3830	* @returns VBox status code.
3831	* @param pVCpu The cross context virtual CPU structure.
3832	* @param pVmcsInfo The VMCS info. object.
3833	*/
3834	static int hmR0VmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3835	{
3836	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3837	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3838	{
3839	hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3840	hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3841	}
3842	#endif
3843
3844	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3845	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3846	AssertRC(rc);
3847
3848	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3849	if (RT_SUCCESS(rc))
3850	{
3851	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3852	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3853
3854	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
3855	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
3856
3857	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3858	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3859	return VINF_SUCCESS;
3860	}
3861	else
3862	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3863	return rc;
3864	}
3865
3866
3867	/**
3868	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3869	*
3870	* We shall setup those exception intercepts that don't change during the
3871	* lifetime of the VM here. The rest are done dynamically while loading the
3872	* guest state.
3873	*
3874	* @param pVCpu The cross context virtual CPU structure.
3875	* @param pVmcsInfo The VMCS info. object.
3876	*/
3877	static void hmR0VmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3878	{
3879	/*
3880	* The following exceptions are always intercepted:
3881	*
3882	* #AC - To prevent the guest from hanging the CPU.
3883	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3884	* recursive #DBs can cause a CPU hang.
3885	* #PF - To sync our shadow page tables when nested-paging is not used.
3886	*/
3887	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3888	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3889	\| RT_BIT(X86_XCPT_DB)
3890	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3891
3892	/* Commit it to the VMCS. */
3893	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3894	AssertRC(rc);
3895
3896	/* Update our cache of the exception bitmap. */
3897	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3898	}
3899
3900
3901	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3902	/**
3903	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3904	*
3905	* @returns VBox status code.
3906	* @param pVCpu The cross context virtual CPU structure.
3907	* @param pVmcsInfo The VMCS info. object.
3908	*/
3909	static int hmR0VmxSetupVmcsCtlsNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
3910	{
3911	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3912	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3913	AssertRC(rc);
3914
3915	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3916	if (RT_SUCCESS(rc))
3917	{
3918	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3919	hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3920
3921	/* Paranoia - We've not yet initialized these, they shall be done while merging the VMCS. */
3922	Assert(!pVmcsInfo->u64Cr0Mask);
3923	Assert(!pVmcsInfo->u64Cr4Mask);
3924	return VINF_SUCCESS;
3925	}
3926	else
3927	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3928	return rc;
3929	}
3930	#endif
3931
3932
3933	/**
3934	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3935	* VMX.
3936	*
3937	* @returns VBox status code.
3938	* @param pVCpu The cross context virtual CPU structure.
3939	* @param pVmcsInfo The VMCS info. object.
3940	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3941	*/
3942	static int hmR0VmxSetupVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3943	{
3944	Assert(pVmcsInfo->pvVmcs);
3945	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3946
3947	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3948	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3949	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3950	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3951
3952	LogFlowFunc(("\n"));
3953
3954	/*
3955	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3956	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3957	*/
3958	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3959	if (RT_SUCCESS(rc))
3960	{
3961	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3962	if (RT_SUCCESS(rc))
3963	{
3964	if (!fIsNstGstVmcs)
3965	{
3966	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3967	if (RT_SUCCESS(rc))
3968	{
3969	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3970	if (RT_SUCCESS(rc))
3971	{
3972	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3973	if (RT_SUCCESS(rc))
3974	{
3975	hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3976	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3977	/*
3978	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
3979	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
3980	* making it fit for use when VMCS shadowing is later enabled.
3981	*/
3982	if (pVmcsInfo->pvShadowVmcs)
3983	{
3984	VMXVMCSREVID VmcsRevId;
3985	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3986	VmcsRevId.n.fIsShadowVmcs = 1;
3987	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
3988	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
3989	if (RT_SUCCESS(rc))
3990	{ /* likely */ }
3991	else
3992	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
3993	}
3994	#endif
3995	}
3996	else
3997	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3998	}
3999	else
4000	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4001	}
4002	else
4003	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4004	}
4005	else
4006	{
4007	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4008	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4009	if (RT_SUCCESS(rc))
4010	{ /* likely */ }
4011	else
4012	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4013	#else
4014	AssertFailed();
4015	#endif
4016	}
4017	}
4018	else
4019	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4020	}
4021	else
4022	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4023
4024	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4025	if (RT_SUCCESS(rc))
4026	{
4027	rc = hmR0VmxClearVmcs(pVmcsInfo);
4028	if (RT_SUCCESS(rc))
4029	{ /* likely */ }
4030	else
4031	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4032	}
4033
4034	/*
4035	* Update the last-error record both for failures and success, so we
4036	* can propagate the status code back to ring-3 for diagnostics.
4037	*/
4038	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4039	NOREF(pszVmcs);
4040	return rc;
4041	}
4042
4043
4044	/**
4045	* Does global VT-x initialization (called during module initialization).
4046	*
4047	* @returns VBox status code.
4048	*/
4049	VMMR0DECL(int) VMXR0GlobalInit(void)
4050	{
4051	#ifdef HMVMX_USE_FUNCTION_TABLE
4052	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4053	# ifdef VBOX_STRICT
4054	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4055	Assert(g_apfnVMExitHandlers[i]);
4056	# endif
4057	#endif
4058	return VINF_SUCCESS;
4059	}
4060
4061
4062	/**
4063	* Does global VT-x termination (called during module termination).
4064	*/
4065	VMMR0DECL(void) VMXR0GlobalTerm()
4066	{
4067	/* Nothing to do currently. */
4068	}
4069
4070
4071	/**
4072	* Sets up and activates VT-x on the current CPU.
4073	*
4074	* @returns VBox status code.
4075	* @param pHostCpu The HM physical-CPU structure.
4076	* @param pVM The cross context VM structure. Can be
4077	* NULL after a host resume operation.
4078	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4079	* fEnabledByHost is @c true).
4080	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4081	* @a fEnabledByHost is @c true).
4082	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4083	* enable VT-x on the host.
4084	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4085	*/
4086	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4087	PCSUPHWVIRTMSRS pHwvirtMsrs)
4088	{
4089	AssertPtr(pHostCpu);
4090	AssertPtr(pHwvirtMsrs);
4091	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4092
4093	/* Enable VT-x if it's not already enabled by the host. */
4094	if (!fEnabledByHost)
4095	{
4096	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4097	if (RT_FAILURE(rc))
4098	return rc;
4099	}
4100
4101	/*
4102	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4103	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4104	* invalidated when flushing by VPID.
4105	*/
4106	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4107	{
4108	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4109	pHostCpu->fFlushAsidBeforeUse = false;
4110	}
4111	else
4112	pHostCpu->fFlushAsidBeforeUse = true;
4113
4114	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4115	++pHostCpu->cTlbFlushes;
4116
4117	return VINF_SUCCESS;
4118	}
4119
4120
4121	/**
4122	* Deactivates VT-x on the current CPU.
4123	*
4124	* @returns VBox status code.
4125	* @param pvCpuPage Pointer to the VMXON region.
4126	* @param HCPhysCpuPage Physical address of the VMXON region.
4127	*
4128	* @remarks This function should never be called when SUPR0EnableVTx() or
4129	* similar was used to enable VT-x on the host.
4130	*/
4131	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4132	{
4133	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4134
4135	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4136	return hmR0VmxLeaveRootMode();
4137	}
4138
4139
4140	/**
4141	* Does per-VM VT-x initialization.
4142	*
4143	* @returns VBox status code.
4144	* @param pVM The cross context VM structure.
4145	*/
4146	VMMR0DECL(int) VMXR0InitVM(PVMCC pVM)
4147	{
4148	AssertPtr(pVM);
4149	LogFlowFunc(("pVM=%p\n", pVM));
4150
4151	int rc = hmR0VmxStructsAlloc(pVM);
4152	if (RT_FAILURE(rc))
4153	{
4154	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4155	return rc;
4156	}
4157
4158	return VINF_SUCCESS;
4159	}
4160
4161
4162	/**
4163	* Does per-VM VT-x termination.
4164	*
4165	* @returns VBox status code.
4166	* @param pVM The cross context VM structure.
4167	*/
4168	VMMR0DECL(int) VMXR0TermVM(PVMCC pVM)
4169	{
4170	AssertPtr(pVM);
4171	LogFlowFunc(("pVM=%p\n", pVM));
4172
4173	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4174	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4175	{
4176	Assert(pVM->hm.s.vmx.pvScratch);
4177	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4178	}
4179	#endif
4180	hmR0VmxStructsFree(pVM);
4181	return VINF_SUCCESS;
4182	}
4183
4184
4185	/**
4186	* Sets up the VM for execution using hardware-assisted VMX.
4187	* This function is only called once per-VM during initialization.
4188	*
4189	* @returns VBox status code.
4190	* @param pVM The cross context VM structure.
4191	*/
4192	VMMR0DECL(int) VMXR0SetupVM(PVMCC pVM)
4193	{
4194	AssertPtr(pVM);
4195	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4196
4197	LogFlowFunc(("pVM=%p\n", pVM));
4198
4199	/*
4200	* At least verify if VMX is enabled, since we can't check if we're in VMX root mode or not
4201	* without causing a #GP.
4202	*/
4203	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4204	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4205	{ /* likely */ }
4206	else
4207	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4208
4209	/*
4210	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4211	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4212	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4213	*/
4214	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4215	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4216	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4217	{
4218	LogRelFunc(("Invalid real-on-v86 state.\n"));
4219	return VERR_INTERNAL_ERROR;
4220	}
4221
4222	/* Initialize these always, see hmR3InitFinalizeR0().*/
4223	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4224	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4225
4226	/* Setup the tagged-TLB flush handlers. */
4227	int rc = hmR0VmxSetupTaggedTlb(pVM);
4228	if (RT_FAILURE(rc))
4229	{
4230	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4231	return rc;
4232	}
4233
4234	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4235	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4236	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4237	{
4238	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4239	if (RT_SUCCESS(rc))
4240	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4241	else
4242	{
4243	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4244	return rc;
4245	}
4246	}
4247	#endif
4248
4249	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4250	{
4251	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
4252	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4253
4254	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4255	if (RT_SUCCESS(rc))
4256	{
4257	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4258	if (pVM->cpum.ro.GuestFeatures.fVmx)
4259	{
4260	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4261	if (RT_SUCCESS(rc))
4262	{ /* likely */ }
4263	else
4264	{
4265	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4266	return rc;
4267	}
4268	}
4269	#endif
4270	}
4271	else
4272	{
4273	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4274	return rc;
4275	}
4276	}
4277
4278	return VINF_SUCCESS;
4279	}
4280
4281
4282	/**
4283	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4284	* the VMCS.
4285	*/
4286	static void hmR0VmxExportHostControlRegs(void)
4287	{
4288	int rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR0, ASMGetCR0()); AssertRC(rc);
4289	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR3, ASMGetCR3()); AssertRC(rc);
4290	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_CR4, ASMGetCR4()); AssertRC(rc);
4291	}
4292
4293
4294	/**
4295	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4296	* the host-state area in the VMCS.
4297	*
4298	* @returns VBox status code.
4299	* @param pVCpu The cross context virtual CPU structure.
4300	*/
4301	static int hmR0VmxExportHostSegmentRegs(PVMCPUCC pVCpu)
4302	{
4303	/**
4304	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4305	* requirements. See hmR0VmxExportHostSegmentRegs().
4306	*/
4307	#define VMXLOCAL_ADJUST_HOST_SEG(a_Seg, a_selValue) \
4308	if ((a_selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4309	{ \
4310	bool fValidSelector = true; \
4311	if ((a_selValue) & X86_SEL_LDT) \
4312	{ \
4313	uint32_t const uAttr = ASMGetSegAttr(a_selValue); \
4314	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4315	} \
4316	if (fValidSelector) \
4317	{ \
4318	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##a_Seg; \
4319	pVCpu->hm.s.vmx.RestoreHost.uHostSel##a_Seg = (a_selValue); \
4320	} \
4321	(a_selValue) = 0; \
4322	}
4323
4324	/*
4325	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4326	* will be messed up. We should -not- save the messed up state without restoring
4327	* the original host-state, see @bugref{7240}.
4328	*
4329	* This apparently can happen (most likely the FPU changes), deal with it rather than
4330	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4331	*/
4332	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4333	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4334	{
4335	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4336	pVCpu->idCpu));
4337	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4338	}
4339	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4340
4341	/*
4342	* Host segment registers.
4343	*/
4344	RTSEL uSelES = ASMGetES();
4345	RTSEL uSelCS = ASMGetCS();
4346	RTSEL uSelSS = ASMGetSS();
4347	RTSEL uSelDS = ASMGetDS();
4348	RTSEL uSelFS = ASMGetFS();
4349	RTSEL uSelGS = ASMGetGS();
4350	RTSEL uSelTR = ASMGetTR();
4351
4352	/*
4353	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4354	* gain VM-entry and restore them before we get preempted.
4355	*
4356	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4357	*/
4358	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4359	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4360	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4361	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4362
4363	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4364	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4365	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4366	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4367	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4368	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4369	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4370	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4371	Assert(uSelCS);
4372	Assert(uSelTR);
4373
4374	/* Write these host selector fields into the host-state area in the VMCS. */
4375	int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS); AssertRC(rc);
4376	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS); AssertRC(rc);
4377	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS); AssertRC(rc);
4378	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES); AssertRC(rc);
4379	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS); AssertRC(rc);
4380	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS); AssertRC(rc);
4381	rc = VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR); AssertRC(rc);
4382
4383	/*
4384	* Host GDTR and IDTR.
4385	*/
4386	RTGDTR Gdtr;
4387	RTIDTR Idtr;
4388	RT_ZERO(Gdtr);
4389	RT_ZERO(Idtr);
4390	ASMGetGDTR(&Gdtr);
4391	ASMGetIDTR(&Idtr);
4392	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRC(rc);
4393	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRC(rc);
4394
4395	/*
4396	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4397	* them to the maximum limit (0xffff) on every VM-exit.
4398	*/
4399	if (Gdtr.cbGdt != 0xffff)
4400	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4401
4402	/*
4403	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4404	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4405	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4406	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4407	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4408	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4409	* at 0xffff on hosts where we are sure it won't cause trouble.
4410	*/
4411	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4412	if (Idtr.cbIdt < 0x0fff)
4413	#else
4414	if (Idtr.cbIdt != 0xffff)
4415	#endif
4416	{
4417	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4418	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4419	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4420	}
4421
4422	/*
4423	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4424	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4425	* RPL should be too in most cases.
4426	*/
4427	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4428	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4429
4430	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4431	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4432
4433	/*
4434	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4435	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4436	* restoration if the host has something else. Task switching is not supported in 64-bit
4437	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4438	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4439	*
4440	* [1] See Intel spec. 3.5 "System Descriptor Types".
4441	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4442	*/
4443	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4444	Assert(pDesc->System.u4Type == 11);
4445	if ( pDesc->System.u16LimitLow != 0x67
4446	\|\| pDesc->System.u4LimitHigh)
4447	{
4448	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4449	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4450	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4451	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4452	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4453	}
4454
4455	/*
4456	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4457	*/
4458	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4459	{
4460	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4461	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4462	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4463	{
4464	/* The GDT is read-only but the writable GDT is available. */
4465	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4466	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4467	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4468	AssertRCReturn(rc, rc);
4469	}
4470	}
4471
4472	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_TR_BASE, uTRBase);
4473	AssertRC(rc);
4474
4475	/*
4476	* Host FS base and GS base.
4477	*/
4478	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4479	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4480	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRC(rc);
4481	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRC(rc);
4482
4483	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4484	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4485	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4486	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4487	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4488
4489	return VINF_SUCCESS;
4490	#undef VMXLOCAL_ADJUST_HOST_SEG
4491	}
4492
4493
4494	/**
4495	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4496	* host-state area of the VMCS.
4497	*
4498	* These MSRs will be automatically restored on the host after every successful
4499	* VM-exit.
4500	*
4501	* @param pVCpu The cross context virtual CPU structure.
4502	*
4503	* @remarks No-long-jump zone!!!
4504	*/
4505	static void hmR0VmxExportHostMsrs(PVMCPUCC pVCpu)
4506	{
4507	AssertPtr(pVCpu);
4508
4509	/*
4510	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4511	* rather than swapping them on every VM-entry.
4512	*/
4513	hmR0VmxLazySaveHostMsrs(pVCpu);
4514
4515	/*
4516	* Host Sysenter MSRs.
4517	*/
4518	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)); AssertRC(rc);
4519	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP)); AssertRC(rc);
4520	rc = VMXWriteVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP)); AssertRC(rc);
4521
4522	/*
4523	* Host EFER MSR.
4524	*
4525	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4526	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4527	*/
4528	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4529	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4530	{
4531	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4532	AssertRC(rc);
4533	}
4534
4535	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4536	* hmR0VmxExportGuestEntryExitCtls(). */
4537	}
4538
4539
4540	/**
4541	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4542	*
4543	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4544	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4545	*
4546	* @returns true if we need to load guest EFER, false otherwise.
4547	* @param pVCpu The cross context virtual CPU structure.
4548	* @param pVmxTransient The VMX-transient structure.
4549	*
4550	* @remarks Requires EFER, CR4.
4551	* @remarks No-long-jump zone!!!
4552	*/
4553	static bool hmR0VmxShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
4554	{
4555	#ifdef HMVMX_ALWAYS_SWAP_EFER
4556	RT_NOREF2(pVCpu, pVmxTransient);
4557	return true;
4558	#else
4559	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4560	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4561	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4562	uint64_t const u64GuestEfer = pCtx->msrEFER;
4563
4564	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4565	/*
4566	* For nested-guests, we shall honor swapping the EFER MSR when requested by
4567	* the nested-guest.
4568	*/
4569	if ( pVmxTransient->fIsNestedGuest
4570	&& ( CPUMIsGuestVmxEntryCtlsSet(pVCpu, pCtx, VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4571	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_SAVE_EFER_MSR)
4572	\|\| CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_LOAD_EFER_MSR)))
4573	return true;
4574	# else
4575	RT_NOREF(pVmxTransient);
4576	#endif
4577
4578	/*
4579	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4580	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4581	*/
4582	if ( CPUMIsGuestInLongModeEx(pCtx)
4583	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4584	return true;
4585
4586	/*
4587	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4588	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4589	*
4590	* See Intel spec. 4.5 "IA-32e Paging".
4591	* See Intel spec. 4.1.1 "Three Paging Modes".
4592	*
4593	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4594	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4595	*/
4596	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4597	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4598	if ( (pCtx->cr4 & X86_CR4_PAE)
4599	&& (pCtx->cr0 & X86_CR0_PG))
4600	{
4601	/*
4602	* If nested paging is not used, verify that the guest paging mode matches the
4603	* shadow paging mode which is/will be placed in the VMCS (which is what will
4604	* actually be used while executing the guest and not the CR4 shadow value).
4605	*/
4606	AssertMsg(pVM->hm.s.fNestedPaging \|\| ( pVCpu->hm.s.enmShadowMode == PGMMODE_PAE
4607	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_PAE_NX
4608	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64
4609	\|\| pVCpu->hm.s.enmShadowMode == PGMMODE_AMD64_NX),
4610	("enmShadowMode=%u\n", pVCpu->hm.s.enmShadowMode));
4611	if ((u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4612	{
4613	/* Verify that the host is NX capable. */
4614	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4615	return true;
4616	}
4617	}
4618
4619	return false;
4620	#endif
4621	}
4622
4623
4624	/**
4625	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4626	* VMCS.
4627	*
4628	* This is typically required when the guest changes paging mode.
4629	*
4630	* @returns VBox status code.
4631	* @param pVCpu The cross context virtual CPU structure.
4632	* @param pVmxTransient The VMX-transient structure.
4633	*
4634	* @remarks Requires EFER.
4635	* @remarks No-long-jump zone!!!
4636	*/
4637	static int hmR0VmxExportGuestEntryExitCtls(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4638	{
4639	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4640	{
4641	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4642	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4643
4644	/*
4645	* VM-entry controls.
4646	*/
4647	{
4648	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4649	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4650
4651	/*
4652	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4653	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4654	*
4655	* For nested-guests, this is a mandatory VM-entry control. It's also
4656	* required because we do not want to leak host bits to the nested-guest.
4657	*/
4658	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4659
4660	/*
4661	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4662	*
4663	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4664	* required to get the nested-guest working with hardware-assisted VMX execution.
4665	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
4666	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4667	* here rather than while merging the guest VMCS controls.
4668	*/
4669	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4670	{
4671	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
4672	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4673	}
4674	else
4675	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4676
4677	/*
4678	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4679	*
4680	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4681	* regardless of whether the nested-guest VMCS specifies it because we are free to
4682	* load whatever MSRs we require and we do not need to modify the guest visible copy
4683	* of the VM-entry MSR load area.
4684	*/
4685	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4686	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4687	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4688	else
4689	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4690
4691	/*
4692	* The following should -not- be set (since we're not in SMM mode):
4693	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4694	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4695	*/
4696
4697	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4698	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4699
4700	if ((fVal & fZap) == fVal)
4701	{ /* likely */ }
4702	else
4703	{
4704	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4705	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4706	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4707	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4708	}
4709
4710	/* Commit it to the VMCS. */
4711	if (pVmcsInfo->u32EntryCtls != fVal)
4712	{
4713	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4714	AssertRC(rc);
4715	pVmcsInfo->u32EntryCtls = fVal;
4716	}
4717	}
4718
4719	/*
4720	* VM-exit controls.
4721	*/
4722	{
4723	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4724	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4725
4726	/*
4727	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4728	* supported the 1-setting of this bit.
4729	*
4730	* For nested-guests, we set the "save debug controls" as the converse
4731	* "load debug controls" is mandatory for nested-guests anyway.
4732	*/
4733	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4734
4735	/*
4736	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4737	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4738	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4739	* hmR0VmxExportHostMsrs().
4740	*
4741	* For nested-guests, we always set this bit as we do not support 32-bit
4742	* hosts.
4743	*/
4744	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4745
4746	/*
4747	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4748	*
4749	* For nested-guests, we should use the "save IA32_EFER" control if we also
4750	* used the "load IA32_EFER" control while exporting VM-entry controls.
4751	*/
4752	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4753	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
4754	{
4755	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4756	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4757	}
4758
4759	/*
4760	* Enable saving of the VMX-preemption timer value on VM-exit.
4761	* For nested-guests, currently not exposed/used.
4762	*/
4763	if ( pVM->hm.s.vmx.fUsePreemptTimer
4764	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4765	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4766
4767	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4768	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4769
4770	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4771	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4772	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4773
4774	if ((fVal & fZap) == fVal)
4775	{ /* likely */ }
4776	else
4777	{
4778	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4779	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4780	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4781	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4782	}
4783
4784	/* Commit it to the VMCS. */
4785	if (pVmcsInfo->u32ExitCtls != fVal)
4786	{
4787	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4788	AssertRC(rc);
4789	pVmcsInfo->u32ExitCtls = fVal;
4790	}
4791	}
4792
4793	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4794	}
4795	return VINF_SUCCESS;
4796	}
4797
4798
4799	/**
4800	* Sets the TPR threshold in the VMCS.
4801	*
4802	* @param pVmcsInfo The VMCS info. object.
4803	* @param u32TprThreshold The TPR threshold (task-priority class only).
4804	*/
4805	DECLINLINE(void) hmR0VmxApicSetTprThreshold(PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4806	{
4807	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4808	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4809	RT_NOREF(pVmcsInfo);
4810	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4811	AssertRC(rc);
4812	}
4813
4814
4815	/**
4816	* Exports the guest APIC TPR state into the VMCS.
4817	*
4818	* @returns VBox status code.
4819	* @param pVCpu The cross context virtual CPU structure.
4820	* @param pVmxTransient The VMX-transient structure.
4821	*
4822	* @remarks No-long-jump zone!!!
4823	*/
4824	static int hmR0VmxExportGuestApicTpr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4825	{
4826	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4827	{
4828	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4829
4830	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4831	if (!pVmxTransient->fIsNestedGuest)
4832	{
4833	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4834	&& APICIsEnabled(pVCpu))
4835	{
4836	/*
4837	* Setup TPR shadowing.
4838	*/
4839	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4840	{
4841	bool fPendingIntr = false;
4842	uint8_t u8Tpr = 0;
4843	uint8_t u8PendingIntr = 0;
4844	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4845	AssertRCReturn(rc, rc);
4846
4847	/*
4848	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4849	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4850	* priority of the pending interrupt so we can deliver the interrupt. If there
4851	* are no interrupts pending, set threshold to 0 to not cause any
4852	* TPR-below-threshold VM-exits.
4853	*/
4854	uint32_t u32TprThreshold = 0;
4855	if (fPendingIntr)
4856	{
4857	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4858	(which is the Task-Priority Class). */
4859	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4860	const uint8_t u8TprPriority = u8Tpr >> 4;
4861	if (u8PendingPriority <= u8TprPriority)
4862	u32TprThreshold = u8PendingPriority;
4863	}
4864
4865	hmR0VmxApicSetTprThreshold(pVmcsInfo, u32TprThreshold);
4866	}
4867	}
4868	}
4869	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4870	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4871	}
4872	return VINF_SUCCESS;
4873	}
4874
4875
4876	/**
4877	* Gets the guest interruptibility-state.
4878	*
4879	* @returns Guest's interruptibility-state.
4880	* @param pVCpu The cross context virtual CPU structure.
4881	* @param pVmxTransient The VMX-transient structure.
4882	*
4883	* @remarks No-long-jump zone!!!
4884	*/
4885	static uint32_t hmR0VmxGetGuestIntrState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4886	{
4887	/*
4888	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4889	*/
4890	uint32_t fIntrState = 0;
4891	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4892	{
4893	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
4894	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
4895
4896	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4897	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4898	{
4899	if (pCtx->eflags.Bits.u1IF)
4900	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4901	else
4902	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4903	}
4904	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4905	{
4906	/*
4907	* We can clear the inhibit force flag as even if we go back to the recompiler
4908	* without executing guest code in VT-x, the flag's condition to be cleared is
4909	* met and thus the cleared state is correct.
4910	*/
4911	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4912	}
4913	}
4914
4915	/*
4916	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4917	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4918	* setting this would block host-NMIs and IRET will not clear the blocking.
4919	*
4920	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4921	*
4922	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4923	*/
4924	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4925	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4926	&& CPUMIsGuestNmiBlocking(pVCpu))
4927	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4928
4929	return fIntrState;
4930	}
4931
4932
4933	/**
4934	* Exports the exception intercepts required for guest execution in the VMCS.
4935	*
4936	* @returns VBox status code.
4937	* @param pVCpu The cross context virtual CPU structure.
4938	* @param pVmxTransient The VMX-transient structure.
4939	*
4940	* @remarks No-long-jump zone!!!
4941	*/
4942	static int hmR0VmxExportGuestXcptIntercepts(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
4943	{
4944	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4945	{
4946	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4947	if ( !pVmxTransient->fIsNestedGuest
4948	&& pVCpu->hm.s.fGIMTrapXcptUD)
4949	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4950	else
4951	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4952
4953	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4954	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4955	}
4956	return VINF_SUCCESS;
4957	}
4958
4959
4960	/**
4961	* Exports the guest's RIP into the guest-state area in the VMCS.
4962	*
4963	* @returns VBox status code.
4964	* @param pVCpu The cross context virtual CPU structure.
4965	*
4966	* @remarks No-long-jump zone!!!
4967	*/
4968	static int hmR0VmxExportGuestRip(PVMCPUCC pVCpu)
4969	{
4970	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4971	{
4972	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4973
4974	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4975	AssertRC(rc);
4976
4977	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4978	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4979	}
4980	return VINF_SUCCESS;
4981	}
4982
4983
4984	/**
4985	* Exports the guest's RSP into the guest-state area in the VMCS.
4986	*
4987	* @returns VBox status code.
4988	* @param pVCpu The cross context virtual CPU structure.
4989	*
4990	* @remarks No-long-jump zone!!!
4991	*/
4992	static int hmR0VmxExportGuestRsp(PVMCPUCC pVCpu)
4993	{
4994	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4995	{
4996	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4997
4998	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4999	AssertRC(rc);
5000
5001	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5002	Log4Func(("rsp=%#RX64\n", pVCpu->cpum.GstCtx.rsp));
5003	}
5004	return VINF_SUCCESS;
5005	}
5006
5007
5008	/**
5009	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5010	*
5011	* @returns VBox status code.
5012	* @param pVCpu The cross context virtual CPU structure.
5013	* @param pVmxTransient The VMX-transient structure.
5014	*
5015	* @remarks No-long-jump zone!!!
5016	*/
5017	static int hmR0VmxExportGuestRflags(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5018	{
5019	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5020	{
5021	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5022
5023	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5024	Let us assert it as such and use 32-bit VMWRITE. */
5025	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5026	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5027	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5028	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5029
5030	/*
5031	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5032	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5033	* can run the real-mode guest code under Virtual 8086 mode.
5034	*/
5035	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5036	if (pVmcsInfo->RealMode.fRealOnV86Active)
5037	{
5038	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5039	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5040	Assert(!pVmxTransient->fIsNestedGuest);
5041	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5042	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5043	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5044	}
5045
5046	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5047	AssertRC(rc);
5048
5049	/*
5050	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5051	*
5052	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5053	* through the hypervisor debugger using EFLAGS.TF.
5054	*/
5055	if ( !pVmxTransient->fIsNestedGuest
5056	&& !pVCpu->hm.s.fSingleInstruction
5057	&& fEFlags.Bits.u1TF)
5058	{
5059	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5060	* premature trips to ring-3 esp since IEM does not yet handle it. */
5061	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5062	AssertRC(rc);
5063	}
5064	/* else: for nested-guest currently handling while merging controls. */
5065
5066	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5067	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5068	}
5069	return VINF_SUCCESS;
5070	}
5071
5072
5073	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5074	/**
5075	* Copies the nested-guest VMCS to the shadow VMCS.
5076	*
5077	* @returns VBox status code.
5078	* @param pVCpu The cross context virtual CPU structure.
5079	* @param pVmcsInfo The VMCS info. object.
5080	*
5081	* @remarks No-long-jump zone!!!
5082	*/
5083	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5084	{
5085	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5086	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5087
5088	/*
5089	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5090	* current VMCS, as we may try saving guest lazy MSRs.
5091	*
5092	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5093	* calling the import VMCS code which is currently performing the guest MSR reads
5094	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5095	* and the rest of the VMX leave session machinery.
5096	*/
5097	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5098
5099	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5100	if (RT_SUCCESS(rc))
5101	{
5102	/*
5103	* Copy all guest read/write VMCS fields.
5104	*
5105	* We don't check for VMWRITE failures here for performance reasons and
5106	* because they are not expected to fail, barring irrecoverable conditions
5107	* like hardware errors.
5108	*/
5109	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5110	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5111	{
5112	uint64_t u64Val;
5113	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5114	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5115	VMXWriteVmcs64(uVmcsField, u64Val);
5116	}
5117
5118	/*
5119	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5120	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5121	*/
5122	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5123	{
5124	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5125	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5126	{
5127	uint64_t u64Val;
5128	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5129	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5130	VMXWriteVmcs64(uVmcsField, u64Val);
5131	}
5132	}
5133
5134	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5135	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5136	}
5137
5138	ASMSetFlags(fEFlags);
5139	return rc;
5140	}
5141
5142
5143	/**
5144	* Copies the shadow VMCS to the nested-guest VMCS.
5145	*
5146	* @returns VBox status code.
5147	* @param pVCpu The cross context virtual CPU structure.
5148	* @param pVmcsInfo The VMCS info. object.
5149	*
5150	* @remarks Called with interrupts disabled.
5151	*/
5152	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5153	{
5154	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5155	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5156	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5157
5158	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5159	if (RT_SUCCESS(rc))
5160	{
5161	/*
5162	* Copy guest read/write fields from the shadow VMCS.
5163	* Guest read-only fields cannot be modified, so no need to copy them.
5164	*
5165	* We don't check for VMREAD failures here for performance reasons and
5166	* because they are not expected to fail, barring irrecoverable conditions
5167	* like hardware errors.
5168	*/
5169	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5170	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5171	{
5172	uint64_t u64Val;
5173	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5174	VMXReadVmcs64(uVmcsField, &u64Val);
5175	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5176	}
5177
5178	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5179	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5180	}
5181	return rc;
5182	}
5183
5184
5185	/**
5186	* Enables VMCS shadowing for the given VMCS info. object.
5187	*
5188	* @param pVmcsInfo The VMCS info. object.
5189	*
5190	* @remarks No-long-jump zone!!!
5191	*/
5192	static void hmR0VmxEnableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5193	{
5194	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5195	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5196	{
5197	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5198	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5199	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5200	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
5201	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5202	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5203	Log4Func(("Enabled\n"));
5204	}
5205	}
5206
5207
5208	/**
5209	* Disables VMCS shadowing for the given VMCS info. object.
5210	*
5211	* @param pVmcsInfo The VMCS info. object.
5212	*
5213	* @remarks No-long-jump zone!!!
5214	*/
5215	static void hmR0VmxDisableVmcsShadowing(PVMXVMCSINFO pVmcsInfo)
5216	{
5217	/*
5218	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5219	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5220	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5221	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5222	*
5223	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5224	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5225	*/
5226	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5227	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5228	{
5229	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5230	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
5231	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
5232	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5233	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5234	Log4Func(("Disabled\n"));
5235	}
5236	}
5237	#endif
5238
5239
5240	/**
5241	* Exports the guest hardware-virtualization state.
5242	*
5243	* @returns VBox status code.
5244	* @param pVCpu The cross context virtual CPU structure.
5245	* @param pVmxTransient The VMX-transient structure.
5246	*
5247	* @remarks No-long-jump zone!!!
5248	*/
5249	static int hmR0VmxExportGuestHwvirtState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5250	{
5251	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5252	{
5253	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5254	/*
5255	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5256	* VMCS shadowing.
5257	*/
5258	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5259	{
5260	/*
5261	* If the nested hypervisor has loaded a current VMCS and is in VMX root mode,
5262	* copy the nested hypervisor's current VMCS into the shadow VMCS and enable
5263	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5264	*
5265	* We check for VMX root mode here in case the guest executes VMXOFF without
5266	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5267	* not clear the current VMCS pointer.
5268	*/
5269	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5270	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5271	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5272	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5273	{
5274	/* Paranoia. */
5275	Assert(!pVmxTransient->fIsNestedGuest);
5276
5277	/*
5278	* For performance reasons, also check if the nested hypervisor's current VMCS
5279	* was newly loaded or modified before copying it to the shadow VMCS.
5280	*/
5281	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5282	{
5283	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5284	AssertRCReturn(rc, rc);
5285	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5286	}
5287	hmR0VmxEnableVmcsShadowing(pVmcsInfo);
5288	}
5289	else
5290	hmR0VmxDisableVmcsShadowing(pVmcsInfo);
5291	}
5292	#else
5293	NOREF(pVmxTransient);
5294	#endif
5295	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5296	}
5297	return VINF_SUCCESS;
5298	}
5299
5300
5301	/**
5302	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5303	*
5304	* The guest FPU state is always pre-loaded hence we don't need to bother about
5305	* sharing FPU related CR0 bits between the guest and host.
5306	*
5307	* @returns VBox status code.
5308	* @param pVCpu The cross context virtual CPU structure.
5309	* @param pVmxTransient The VMX-transient structure.
5310	*
5311	* @remarks No-long-jump zone!!!
5312	*/
5313	static int hmR0VmxExportGuestCR0(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5314	{
5315	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5316	{
5317	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5318	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5319
5320	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0;
5321	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5322	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5323	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5324	else
5325	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5326
5327	if (!pVmxTransient->fIsNestedGuest)
5328	{
5329	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5330	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5331	uint64_t const u64ShadowCr0 = u64GuestCr0;
5332	Assert(!RT_HI_U32(u64GuestCr0));
5333
5334	/*
5335	* Setup VT-x's view of the guest CR0.
5336	*/
5337	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5338	if (pVM->hm.s.fNestedPaging)
5339	{
5340	if (CPUMIsGuestPagingEnabled(pVCpu))
5341	{
5342	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5343	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5344	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5345	}
5346	else
5347	{
5348	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5349	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5350	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5351	}
5352
5353	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5354	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5355	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5356	}
5357	else
5358	{
5359	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5360	u64GuestCr0 \|= X86_CR0_WP;
5361	}
5362
5363	/*
5364	* Guest FPU bits.
5365	*
5366	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5367	* using CR0.TS.
5368	*
5369	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5370	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5371	*/
5372	u64GuestCr0 \|= X86_CR0_NE;
5373
5374	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5375	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5376
5377	/*
5378	* Update exception intercepts.
5379	*/
5380	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5381	if (pVmcsInfo->RealMode.fRealOnV86Active)
5382	{
5383	Assert(PDMVmmDevHeapIsEnabled(pVM));
5384	Assert(pVM->hm.s.vmx.pRealModeTSS);
5385	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5386	}
5387	else
5388	{
5389	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5390	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5391	if (fInterceptMF)
5392	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5393	}
5394
5395	/* Additional intercepts for debugging, define these yourself explicitly. */
5396	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5397	uXcptBitmap \|= 0
5398	\| RT_BIT(X86_XCPT_BP)
5399	\| RT_BIT(X86_XCPT_DE)
5400	\| RT_BIT(X86_XCPT_NM)
5401	\| RT_BIT(X86_XCPT_TS)
5402	\| RT_BIT(X86_XCPT_UD)
5403	\| RT_BIT(X86_XCPT_NP)
5404	\| RT_BIT(X86_XCPT_SS)
5405	\| RT_BIT(X86_XCPT_GP)
5406	\| RT_BIT(X86_XCPT_PF)
5407	\| RT_BIT(X86_XCPT_MF)
5408	;
5409	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5410	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5411	#endif
5412	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5413	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5414	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5415
5416	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5417	u64GuestCr0 \|= fSetCr0;
5418	u64GuestCr0 &= fZapCr0;
5419	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5420
5421	/* Commit the CR0 and related fields to the guest VMCS. */
5422	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5423	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5424	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5425	{
5426	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5427	AssertRC(rc);
5428	}
5429	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5430	{
5431	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5432	AssertRC(rc);
5433	}
5434
5435	/* Update our caches. */
5436	pVmcsInfo->u32ProcCtls = uProcCtls;
5437	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5438
5439	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5440	}
5441	else
5442	{
5443	/*
5444	* With nested-guests, we may have extended the guest/host mask here since we
5445	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5446	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
5447	* originally supplied. We must copy those bits from the nested-guest CR0 into
5448	* the nested-guest CR0 read-shadow.
5449	*/
5450	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5451	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5452	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5453	Assert(!RT_HI_U32(u64GuestCr0));
5454	Assert(u64GuestCr0 & X86_CR0_NE);
5455
5456	/* Apply the hardware specified CR0 fixed bits and enable caching. */
5457	u64GuestCr0 \|= fSetCr0;
5458	u64GuestCr0 &= fZapCr0;
5459	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5460
5461	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5462	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
5463	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
5464
5465	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5466	}
5467
5468	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5469	}
5470
5471	return VINF_SUCCESS;
5472	}
5473
5474
5475	/**
5476	* Exports the guest control registers (CR3, CR4) into the guest-state area
5477	* in the VMCS.
5478	*
5479	* @returns VBox strict status code.
5480	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5481	* without unrestricted guest access and the VMMDev is not presently
5482	* mapped (e.g. EFI32).
5483	*
5484	* @param pVCpu The cross context virtual CPU structure.
5485	* @param pVmxTransient The VMX-transient structure.
5486	*
5487	* @remarks No-long-jump zone!!!
5488	*/
5489	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5490	{
5491	int rc = VINF_SUCCESS;
5492	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5493
5494	/*
5495	* Guest CR2.
5496	* It's always loaded in the assembler code. Nothing to do here.
5497	*/
5498
5499	/*
5500	* Guest CR3.
5501	*/
5502	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5503	{
5504	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5505
5506	if (pVM->hm.s.fNestedPaging)
5507	{
5508	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5509	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5510
5511	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5512	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5513	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5514	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5515
5516	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5517	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5518	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5519
5520	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5521	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5522	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5523	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5524	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5525	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5526	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5527
5528	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5529	AssertRC(rc);
5530
5531	uint64_t u64GuestCr3;
5532	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5533	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5534	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5535	{
5536	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5537	if (CPUMIsGuestInPAEModeEx(pCtx))
5538	{
5539	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5540	AssertRC(rc);
5541	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
5542	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
5543	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
5544	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
5545	}
5546
5547	/*
5548	* The guest's view of its CR3 is unblemished with nested paging when the
5549	* guest is using paging or we have unrestricted guest execution to handle
5550	* the guest when it's not using paging.
5551	*/
5552	u64GuestCr3 = pCtx->cr3;
5553	}
5554	else
5555	{
5556	/*
5557	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5558	* thinks it accesses physical memory directly, we use our identity-mapped
5559	* page table to map guest-linear to guest-physical addresses. EPT takes care
5560	* of translating it to host-physical addresses.
5561	*/
5562	RTGCPHYS GCPhys;
5563	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5564
5565	/* We obtain it here every time as the guest could have relocated this PCI region. */
5566	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5567	if (RT_SUCCESS(rc))
5568	{ /* likely */ }
5569	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5570	{
5571	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5572	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5573	}
5574	else
5575	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5576
5577	u64GuestCr3 = GCPhys;
5578	}
5579
5580	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
5581	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, u64GuestCr3);
5582	AssertRC(rc);
5583	}
5584	else
5585	{
5586	Assert(!pVmxTransient->fIsNestedGuest);
5587	/* Non-nested paging case, just use the hypervisor's CR3. */
5588	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5589
5590	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
5591	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5592	AssertRC(rc);
5593	}
5594
5595	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5596	}
5597
5598	/*
5599	* Guest CR4.
5600	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5601	*/
5602	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5603	{
5604	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5605	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5606
5607	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0;
5608	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5609
5610	/*
5611	* With nested-guests, we may have extended the guest/host mask here (since we
5612	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5613	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5614	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
5615	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5616	*/
5617	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5618	uint64_t u64GuestCr4 = pCtx->cr4;
5619	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5620	? pCtx->cr4
5621	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5622	Assert(!RT_HI_U32(u64GuestCr4));
5623
5624	/*
5625	* Setup VT-x's view of the guest CR4.
5626	*
5627	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5628	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5629	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5630	*
5631	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5632	*/
5633	if (pVmcsInfo->RealMode.fRealOnV86Active)
5634	{
5635	Assert(pVM->hm.s.vmx.pRealModeTSS);
5636	Assert(PDMVmmDevHeapIsEnabled(pVM));
5637	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5638	}
5639
5640	if (pVM->hm.s.fNestedPaging)
5641	{
5642	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5643	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5644	{
5645	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5646	u64GuestCr4 \|= X86_CR4_PSE;
5647	/* Our identity mapping is a 32-bit page directory. */
5648	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5649	}
5650	/* else use guest CR4.*/
5651	}
5652	else
5653	{
5654	Assert(!pVmxTransient->fIsNestedGuest);
5655
5656	/*
5657	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5658	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5659	*/
5660	switch (pVCpu->hm.s.enmShadowMode)
5661	{
5662	case PGMMODE_REAL: /* Real-mode. */
5663	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5664	case PGMMODE_32_BIT: /* 32-bit paging. */
5665	{
5666	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5667	break;
5668	}
5669
5670	case PGMMODE_PAE: /* PAE paging. */
5671	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5672	{
5673	u64GuestCr4 \|= X86_CR4_PAE;
5674	break;
5675	}
5676
5677	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5678	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5679	{
5680	#ifdef VBOX_WITH_64_BITS_GUESTS
5681	/* For our assumption in hmR0VmxShouldSwapEferMsr. */
5682	Assert(u64GuestCr4 & X86_CR4_PAE);
5683	break;
5684	#endif
5685	}
5686	default:
5687	AssertFailed();
5688	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5689	}
5690	}
5691
5692	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
5693	u64GuestCr4 \|= fSetCr4;
5694	u64GuestCr4 &= fZapCr4;
5695
5696	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5697	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
5698	rc = VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
5699
5700	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5701	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5702
5703	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5704
5705	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5706	}
5707	return rc;
5708	}
5709
5710
5711	/**
5712	* Exports the guest debug registers into the guest-state area in the VMCS.
5713	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5714	*
5715	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5716	*
5717	* @returns VBox status code.
5718	* @param pVCpu The cross context virtual CPU structure.
5719	* @param pVmxTransient The VMX-transient structure.
5720	*
5721	* @remarks No-long-jump zone!!!
5722	*/
5723	static int hmR0VmxExportSharedDebugState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5724	{
5725	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5726
5727	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5728	* stepping. */
5729	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5730	if (pVmxTransient->fIsNestedGuest)
5731	{
5732	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5733	AssertRC(rc);
5734
5735	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5736	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5737	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5738	AssertRC(rc);
5739	return VINF_SUCCESS;
5740	}
5741
5742	#ifdef VBOX_STRICT
5743	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5744	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5745	{
5746	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5747	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5748	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5749	}
5750	#endif
5751
5752	bool fSteppingDB = false;
5753	bool fInterceptMovDRx = false;
5754	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5755	if (pVCpu->hm.s.fSingleInstruction)
5756	{
5757	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5758	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5759	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5760	{
5761	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5762	Assert(fSteppingDB == false);
5763	}
5764	else
5765	{
5766	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5767	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5768	pVCpu->hm.s.fClearTrapFlag = true;
5769	fSteppingDB = true;
5770	}
5771	}
5772
5773	uint64_t u64GuestDr7;
5774	if ( fSteppingDB
5775	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5776	{
5777	/*
5778	* Use the combined guest and host DRx values found in the hypervisor register set
5779	* because the hypervisor debugger has breakpoints active or someone is single stepping
5780	* on the host side without a monitor trap flag.
5781	*
5782	* Note! DBGF expects a clean DR6 state before executing guest code.
5783	*/
5784	if (!CPUMIsHyperDebugStateActive(pVCpu))
5785	{
5786	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5787	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5788	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5789	}
5790
5791	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5792	u64GuestDr7 = CPUMGetHyperDR7(pVCpu);
5793	pVCpu->hm.s.fUsingHyperDR7 = true;
5794	fInterceptMovDRx = true;
5795	}
5796	else
5797	{
5798	/*
5799	* If the guest has enabled debug registers, we need to load them prior to
5800	* executing guest code so they'll trigger at the right time.
5801	*/
5802	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5803	{
5804	if (!CPUMIsGuestDebugStateActive(pVCpu))
5805	{
5806	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5807	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5808	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5809	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5810	}
5811	Assert(!fInterceptMovDRx);
5812	}
5813	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5814	{
5815	/*
5816	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5817	* must intercept #DB in order to maintain a correct DR6 guest value, and
5818	* because we need to intercept it to prevent nested #DBs from hanging the
5819	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5820	*/
5821	fInterceptMovDRx = true;
5822	}
5823
5824	/* Update DR7 with the actual guest value. */
5825	u64GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5826	pVCpu->hm.s.fUsingHyperDR7 = false;
5827	}
5828
5829	if (fInterceptMovDRx)
5830	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5831	else
5832	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5833
5834	/*
5835	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5836	* monitor-trap flag and update our cache.
5837	*/
5838	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5839	{
5840	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5841	AssertRC(rc);
5842	pVmcsInfo->u32ProcCtls = uProcCtls;
5843	}
5844
5845	/*
5846	* Update guest DR7.
5847	*/
5848	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, u64GuestDr7);
5849	AssertRC(rc);
5850
5851	/*
5852	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5853	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5854	*
5855	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5856	*/
5857	if (fSteppingDB)
5858	{
5859	Assert(pVCpu->hm.s.fSingleInstruction);
5860	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5861
5862	uint32_t fIntrState = 0;
5863	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5864	AssertRC(rc);
5865
5866	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5867	{
5868	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5869	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5870	AssertRC(rc);
5871	}
5872	}
5873
5874	return VINF_SUCCESS;
5875	}
5876
5877
5878	#ifdef VBOX_STRICT
5879	/**
5880	* Strict function to validate segment registers.
5881	*
5882	* @param pVCpu The cross context virtual CPU structure.
5883	* @param pVmcsInfo The VMCS info. object.
5884	*
5885	* @remarks Will import guest CR0 on strict builds during validation of
5886	* segments.
5887	*/
5888	static void hmR0VmxValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
5889	{
5890	/*
5891	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5892	*
5893	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5894	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5895	* unusable bit and doesn't change the guest-context value.
5896	*/
5897	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5898	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5899	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5900	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5901	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5902	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5903	{
5904	/* Protected mode checks */
5905	/* CS */
5906	Assert(pCtx->cs.Attr.n.u1Present);
5907	Assert(!(pCtx->cs.Attr.u & 0xf00));
5908	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5909	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5910	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5911	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5912	\|\| (pCtx->cs.Attr.n.u1Granularity));
5913	/* CS cannot be loaded with NULL in protected mode. */
5914	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5915	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5916	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5917	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5918	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5919	else
5920	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5921	/* SS */
5922	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5923	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5924	if ( !(pCtx->cr0 & X86_CR0_PE)
5925	\|\| pCtx->cs.Attr.n.u4Type == 3)
5926	{
5927	Assert(!pCtx->ss.Attr.n.u2Dpl);
5928	}
5929	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5930	{
5931	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5932	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5933	Assert(pCtx->ss.Attr.n.u1Present);
5934	Assert(!(pCtx->ss.Attr.u & 0xf00));
5935	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5936	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5937	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5938	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5939	\|\| (pCtx->ss.Attr.n.u1Granularity));
5940	}
5941	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5942	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5943	{
5944	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5945	Assert(pCtx->ds.Attr.n.u1Present);
5946	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5947	Assert(!(pCtx->ds.Attr.u & 0xf00));
5948	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5949	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5950	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5951	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5952	\|\| (pCtx->ds.Attr.n.u1Granularity));
5953	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5954	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5955	}
5956	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5957	{
5958	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5959	Assert(pCtx->es.Attr.n.u1Present);
5960	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5961	Assert(!(pCtx->es.Attr.u & 0xf00));
5962	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5963	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5964	\|\| !(pCtx->es.Attr.n.u1Granularity));
5965	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5966	\|\| (pCtx->es.Attr.n.u1Granularity));
5967	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5968	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5969	}
5970	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5971	{
5972	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5973	Assert(pCtx->fs.Attr.n.u1Present);
5974	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5975	Assert(!(pCtx->fs.Attr.u & 0xf00));
5976	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5977	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5978	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5979	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5980	\|\| (pCtx->fs.Attr.n.u1Granularity));
5981	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5982	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5983	}
5984	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5985	{
5986	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5987	Assert(pCtx->gs.Attr.n.u1Present);
5988	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5989	Assert(!(pCtx->gs.Attr.u & 0xf00));
5990	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5991	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5992	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5993	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5994	\|\| (pCtx->gs.Attr.n.u1Granularity));
5995	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5996	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5997	}
5998	/* 64-bit capable CPUs. */
5999	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6000	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6001	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6002	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6003	}
6004	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6005	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6006	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6007	{
6008	/* Real and v86 mode checks. */
6009	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6010	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6011	if (pVmcsInfo->RealMode.fRealOnV86Active)
6012	{
6013	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6014	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6015	}
6016	else
6017	{
6018	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6019	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6020	}
6021
6022	/* CS */
6023	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6024	Assert(pCtx->cs.u32Limit == 0xffff);
6025	Assert(u32CSAttr == 0xf3);
6026	/* SS */
6027	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6028	Assert(pCtx->ss.u32Limit == 0xffff);
6029	Assert(u32SSAttr == 0xf3);
6030	/* DS */
6031	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6032	Assert(pCtx->ds.u32Limit == 0xffff);
6033	Assert(u32DSAttr == 0xf3);
6034	/* ES */
6035	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6036	Assert(pCtx->es.u32Limit == 0xffff);
6037	Assert(u32ESAttr == 0xf3);
6038	/* FS */
6039	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6040	Assert(pCtx->fs.u32Limit == 0xffff);
6041	Assert(u32FSAttr == 0xf3);
6042	/* GS */
6043	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6044	Assert(pCtx->gs.u32Limit == 0xffff);
6045	Assert(u32GSAttr == 0xf3);
6046	/* 64-bit capable CPUs. */
6047	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6048	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6049	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6050	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6051	}
6052	}
6053	#endif /* VBOX_STRICT */
6054
6055
6056	/**
6057	* Exports a guest segment register into the guest-state area in the VMCS.
6058	*
6059	* @returns VBox status code.
6060	* @param pVCpu The cross context virtual CPU structure.
6061	* @param pVmcsInfo The VMCS info. object.
6062	* @param iSegReg The segment register number (X86_SREG_XXX).
6063	* @param pSelReg Pointer to the segment selector.
6064	*
6065	* @remarks No-long-jump zone!!!
6066	*/
6067	static int hmR0VmxExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6068	{
6069	Assert(iSegReg < X86_SREG_COUNT);
6070	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6071	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6072	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6073	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6074
6075	uint32_t u32Access = pSelReg->Attr.u;
6076	if (pVmcsInfo->RealMode.fRealOnV86Active)
6077	{
6078	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6079	u32Access = 0xf3;
6080	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6081	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6082	RT_NOREF_PV(pVCpu);
6083	}
6084	else
6085	{
6086	/*
6087	* The way to differentiate between whether this is really a null selector or was just
6088	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6089	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6090	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6091	* NULL selectors loaded in protected-mode have their attribute as 0.
6092	*/
6093	if (!u32Access)
6094	u32Access = X86DESCATTR_UNUSABLE;
6095	}
6096
6097	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6098	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6099	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6100
6101	/*
6102	* Commit it to the VMCS.
6103	*/
6104	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); AssertRC(rc);
6105	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); AssertRC(rc);
6106	rc = VMXWriteVmcsNw(idxBase, pSelReg->u64Base); AssertRC(rc);
6107	rc = VMXWriteVmcs32(idxAttr, u32Access); AssertRC(rc);
6108	return VINF_SUCCESS;
6109	}
6110
6111
6112	/**
6113	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6114	* area in the VMCS.
6115	*
6116	* @returns VBox status code.
6117	* @param pVCpu The cross context virtual CPU structure.
6118	* @param pVmxTransient The VMX-transient structure.
6119	*
6120	* @remarks Will import guest CR0 on strict builds during validation of
6121	* segments.
6122	* @remarks No-long-jump zone!!!
6123	*/
6124	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6125	{
6126	int rc = VERR_INTERNAL_ERROR_5;
6127	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6128	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6129	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6130
6131	/*
6132	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6133	*/
6134	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6135	{
6136	#ifdef VBOX_WITH_REM
6137	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6138	{
6139	Assert(!pVmxTransient->fIsNestedGuest);
6140	Assert(pVM->hm.s.vmx.pRealModeTSS);
6141	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6142	if ( pVmcsInfo->fWasInRealMode
6143	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6144	{
6145	/*
6146	* Notify the recompiler must flush its code-cache as the guest -may-
6147	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6148	*/
6149	REMFlushTBs(pVM);
6150	Log4Func(("Switch to protected mode detected!\n"));
6151	pVmcsInfo->fWasInRealMode = false;
6152	}
6153	}
6154	#endif
6155	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6156	{
6157	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6158	if (pVmcsInfo->RealMode.fRealOnV86Active)
6159	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6160	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6161	AssertRC(rc);
6162	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6163	}
6164
6165	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6166	{
6167	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6168	if (pVmcsInfo->RealMode.fRealOnV86Active)
6169	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6170	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6171	AssertRC(rc);
6172	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6173	}
6174
6175	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6176	{
6177	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6178	if (pVmcsInfo->RealMode.fRealOnV86Active)
6179	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6180	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6181	AssertRC(rc);
6182	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6183	}
6184
6185	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6186	{
6187	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6188	if (pVmcsInfo->RealMode.fRealOnV86Active)
6189	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6190	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6191	AssertRC(rc);
6192	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6193	}
6194
6195	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6196	{
6197	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6198	if (pVmcsInfo->RealMode.fRealOnV86Active)
6199	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6200	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6201	AssertRC(rc);
6202	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6203	}
6204
6205	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6206	{
6207	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6208	if (pVmcsInfo->RealMode.fRealOnV86Active)
6209	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6210	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6211	AssertRC(rc);
6212	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6213	}
6214
6215	#ifdef VBOX_STRICT
6216	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6217	#endif
6218	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6219	pCtx->cs.Attr.u));
6220	}
6221
6222	/*
6223	* Guest TR.
6224	*/
6225	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6226	{
6227	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6228
6229	/*
6230	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6231	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6232	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6233	*/
6234	uint16_t u16Sel;
6235	uint32_t u32Limit;
6236	uint64_t u64Base;
6237	uint32_t u32AccessRights;
6238	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6239	{
6240	u16Sel = pCtx->tr.Sel;
6241	u32Limit = pCtx->tr.u32Limit;
6242	u64Base = pCtx->tr.u64Base;
6243	u32AccessRights = pCtx->tr.Attr.u;
6244	}
6245	else
6246	{
6247	Assert(!pVmxTransient->fIsNestedGuest);
6248	Assert(pVM->hm.s.vmx.pRealModeTSS);
6249	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6250
6251	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6252	RTGCPHYS GCPhys;
6253	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6254	AssertRCReturn(rc, rc);
6255
6256	X86DESCATTR DescAttr;
6257	DescAttr.u = 0;
6258	DescAttr.n.u1Present = 1;
6259	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6260
6261	u16Sel = 0;
6262	u32Limit = HM_VTX_TSS_SIZE;
6263	u64Base = GCPhys;
6264	u32AccessRights = DescAttr.u;
6265	}
6266
6267	/* Validate. */
6268	Assert(!(u16Sel & RT_BIT(2)));
6269	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6270	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6271	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6272	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6273	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6274	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6275	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6276	Assert( (u32Limit & 0xfff) == 0xfff
6277	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6278	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6279	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6280
6281	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
6282	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
6283	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
6284	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
6285
6286	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6287	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6288	}
6289
6290	/*
6291	* Guest GDTR.
6292	*/
6293	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6294	{
6295	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6296
6297	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
6298	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
6299
6300	/* Validate. */
6301	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6302
6303	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6304	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6305	}
6306
6307	/*
6308	* Guest LDTR.
6309	*/
6310	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6311	{
6312	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6313
6314	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6315	uint32_t u32Access;
6316	if ( !pVmxTransient->fIsNestedGuest
6317	&& !pCtx->ldtr.Attr.u)
6318	u32Access = X86DESCATTR_UNUSABLE;
6319	else
6320	u32Access = pCtx->ldtr.Attr.u;
6321
6322	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
6323	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
6324	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
6325	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
6326
6327	/* Validate. */
6328	if (!(u32Access & X86DESCATTR_UNUSABLE))
6329	{
6330	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6331	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6332	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6333	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6334	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6335	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6336	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6337	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6338	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6339	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6340	}
6341
6342	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6343	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6344	}
6345
6346	/*
6347	* Guest IDTR.
6348	*/
6349	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6350	{
6351	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6352
6353	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
6354	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
6355
6356	/* Validate. */
6357	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6358
6359	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6360	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6361	}
6362
6363	return VINF_SUCCESS;
6364	}
6365
6366
6367	/**
6368	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6369	* areas.
6370	*
6371	* These MSRs will automatically be loaded to the host CPU on every successful
6372	* VM-entry and stored from the host CPU on every successful VM-exit.
6373	*
6374	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6375	* actual host MSR values are not- updated here for performance reasons. See
6376	* hmR0VmxExportHostMsrs().
6377	*
6378	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6379	*
6380	* @returns VBox status code.
6381	* @param pVCpu The cross context virtual CPU structure.
6382	* @param pVmxTransient The VMX-transient structure.
6383	*
6384	* @remarks No-long-jump zone!!!
6385	*/
6386	static int hmR0VmxExportGuestMsrs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6387	{
6388	AssertPtr(pVCpu);
6389	AssertPtr(pVmxTransient);
6390
6391	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6392	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6393
6394	/*
6395	* MSRs that we use the auto-load/store MSR area in the VMCS.
6396	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(),
6397	* nothing to do here. The host MSR values are updated when it's safe in
6398	* hmR0VmxLazySaveHostMsrs().
6399	*
6400	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6401	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6402	* emulation. The merged MSR permission bitmap will ensure that we get VM-exits
6403	* for any MSR that are not part of the lazy MSRs so we do not need to place
6404	* those MSRs into the auto-load/store MSR area. Nothing to do here.
6405	*/
6406	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6407	{
6408	/* No auto-load/store MSRs currently. */
6409	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6410	}
6411
6412	/*
6413	* Guest Sysenter MSRs.
6414	*/
6415	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6416	{
6417	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6418
6419	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6420	{
6421	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6422	AssertRC(rc);
6423	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6424	}
6425
6426	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6427	{
6428	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6429	AssertRC(rc);
6430	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6431	}
6432
6433	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6434	{
6435	int rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6436	AssertRC(rc);
6437	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6438	}
6439	}
6440
6441	/*
6442	* Guest/host EFER MSR.
6443	*/
6444	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6445	{
6446	/* Whether we are using the VMCS to swap the EFER MSR must have been
6447	determined earlier while exporting VM-entry/VM-exit controls. */
6448	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6449	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6450
6451	if (hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
6452	{
6453	/*
6454	* EFER.LME is written by software, while EFER.LMA is set by the CPU to (CR0.PG & EFER.LME).
6455	* This means a guest can set EFER.LME=1 while CR0.PG=0 and EFER.LMA can remain 0.
6456	* VT-x requires that "IA-32e mode guest" VM-entry control must be identical to EFER.LMA
6457	* and to CR0.PG. Without unrestricted execution, CR0.PG (used for VT-x, not the shadow)
6458	* must always be 1. This forces us to effectively clear both EFER.LMA and EFER.LME until
6459	* the guest has also set CR0.PG=1. Otherwise, we would run into an invalid-guest state
6460	* during VM-entry.
6461	*/
6462	uint64_t uGuestEferMsr = pCtx->msrEFER;
6463	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6464	{
6465	if (!(pCtx->msrEFER & MSR_K6_EFER_LMA))
6466	uGuestEferMsr &= ~MSR_K6_EFER_LME;
6467	else
6468	Assert((pCtx->msrEFER & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
6469	}
6470
6471	/*
6472	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6473	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6474	*/
6475	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6476	{
6477	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, uGuestEferMsr);
6478	AssertRC(rc);
6479	}
6480	else
6481	{
6482	/*
6483	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6484	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6485	*/
6486	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, uGuestEferMsr,
6487	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6488	AssertRCReturn(rc, rc);
6489	}
6490
6491	Log4Func(("efer=%#RX64 shadow=%#RX64\n", uGuestEferMsr, pCtx->msrEFER));
6492	}
6493	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6494	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6495
6496	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6497	}
6498
6499	/*
6500	* Other MSRs.
6501	* Speculation Control (R/W).
6502	*/
6503	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6504	{
6505	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6506	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6507	{
6508	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6509	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6510	AssertRCReturn(rc, rc);
6511	}
6512	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6513	}
6514
6515	return VINF_SUCCESS;
6516	}
6517
6518
6519	/**
6520	* Selects up the appropriate function to run guest code.
6521	*
6522	* @returns VBox status code.
6523	* @param pVCpu The cross context virtual CPU structure.
6524	* @param pVmxTransient The VMX-transient structure.
6525	*
6526	* @remarks No-long-jump zone!!!
6527	*/
6528	static int hmR0VmxSelectVMRunHandler(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6529	{
6530	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6531	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6532
6533	if (CPUMIsGuestInLongModeEx(pCtx))
6534	{
6535	#ifndef VBOX_WITH_64_BITS_GUESTS
6536	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6537	#else
6538	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6539	/* Guest is in long mode, use the 64-bit handler (host is 64-bit). */
6540	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6541	#endif
6542	}
6543	else
6544	{
6545	/* Guest is not in long mode, use the 32-bit handler. */
6546	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6547	}
6548	Assert(pVmcsInfo->pfnStartVM);
6549	return VINF_SUCCESS;
6550	}
6551
6552
6553	/**
6554	* Wrapper for running the guest code in VT-x.
6555	*
6556	* @returns VBox status code, no informational status codes.
6557	* @param pVCpu The cross context virtual CPU structure.
6558	* @param pVmxTransient The VMX-transient structure.
6559	*
6560	* @remarks No-long-jump zone!!!
6561	*/
6562	DECLINLINE(int) hmR0VmxRunGuest(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
6563	{
6564	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6565	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6566	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6567
6568	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6569
6570	/*
6571	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6572	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6573	* callee-saved and thus the need for this XMM wrapper.
6574	*
6575	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6576	*/
6577	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6578	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6579	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6580	#ifdef VBOX_WITH_KERNEL_USING_XMM
6581	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6582	#else
6583	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, NULL /pvUnused/, pVM, pVCpu);
6584	#endif
6585	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6586	return rc;
6587	}
6588
6589
6590	/**
6591	* Reports world-switch error and dumps some useful debug info.
6592	*
6593	* @param pVCpu The cross context virtual CPU structure.
6594	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6595	* @param pVmxTransient The VMX-transient structure (only
6596	* exitReason updated).
6597	*/
6598	static void hmR0VmxReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6599	{
6600	Assert(pVCpu);
6601	Assert(pVmxTransient);
6602	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6603
6604	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6605	switch (rcVMRun)
6606	{
6607	case VERR_VMX_INVALID_VMXON_PTR:
6608	AssertFailed();
6609	break;
6610	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6611	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6612	{
6613	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6614	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6615	AssertRC(rc);
6616	hmR0VmxReadExitQualVmcs(pVmxTransient);
6617
6618	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6619	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6620	Cannot do it here as we may have been long preempted. */
6621
6622	#ifdef VBOX_STRICT
6623	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6624	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6625	pVmxTransient->uExitReason));
6626	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6627	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6628	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6629	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6630	else
6631	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6632	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6633	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6634
6635	static struct
6636	{
6637	/** Name of the field to log. */
6638	const char *pszName;
6639	/** The VMCS field. */
6640	uint32_t uVmcsField;
6641	/** Whether host support of this field needs to be checked. */
6642	bool fCheckSupport;
6643	} const s_aVmcsFields[] =
6644	{
6645	{ "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false },
6646	{ "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false },
6647	{ "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true },
6648	{ "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false },
6649	{ "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false },
6650	{ "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false },
6651	{ "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false },
6652	{ "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false },
6653	{ "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false },
6654	{ "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false },
6655	{ "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false },
6656	{ "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false },
6657	{ "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false },
6658	{ "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false },
6659	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false },
6660	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false },
6661	{ "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false },
6662	{ "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false },
6663	{ "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false },
6664	{ "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false },
6665	{ "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true },
6666	{ "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false },
6667	{ "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false },
6668	{ "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false },
6669	{ "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
6670	{ "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false },
6671	{ "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false },
6672	{ "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false },
6673	/* The order of selector fields below are fixed! */
6674	{ "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false },
6675	{ "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false },
6676	{ "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false },
6677	{ "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false },
6678	{ "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false },
6679	{ "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false },
6680	{ "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false },
6681	/* End of ordered selector fields. */
6682	{ "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false },
6683	{ "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false },
6684	{ "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false },
6685	{ "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false },
6686	{ "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false },
6687	{ "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false },
6688	{ "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false },
6689	{ "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false }
6690	};
6691
6692	RTGDTR HostGdtr;
6693	ASMGetGDTR(&HostGdtr);
6694
6695	uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
6696	for (uint32_t i = 0; i < cVmcsFields; i++)
6697	{
6698	uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
6699
6700	bool fSupported;
6701	if (!s_aVmcsFields[i].fCheckSupport)
6702	fSupported = true;
6703	else
6704	{
6705	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6706	switch (uVmcsField)
6707	{
6708	case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hm.s.fNestedPaging; break;
6709	case VMX_VMCS16_VPID: fSupported = pVM->hm.s.vmx.fVpid; break;
6710	case VMX_VMCS32_CTRL_PROC_EXEC2:
6711	fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6712	break;
6713	default:
6714	AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
6715	}
6716	}
6717
6718	if (fSupported)
6719	{
6720	uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
6721	switch (uWidth)
6722	{
6723	case VMX_VMCSFIELD_WIDTH_16BIT:
6724	{
6725	uint16_t u16Val;
6726	rc = VMXReadVmcs16(uVmcsField, &u16Val);
6727	AssertRC(rc);
6728	Log4(("%-40s = %#RX16\n", s_aVmcsFields[i].pszName, u16Val));
6729
6730	if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
6731	&& uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
6732	{
6733	if (u16Val < HostGdtr.cbGdt)
6734	{
6735	/* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
6736	static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
6737	"Host FS", "Host GS", "Host TR" };
6738	uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
6739	Assert(idxSel < RT_ELEMENTS(s_apszSel));
6740	PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
6741	hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
6742	}
6743	else
6744	Log4((" Selector value exceeds GDT limit!\n"));
6745	}
6746	break;
6747	}
6748
6749	case VMX_VMCSFIELD_WIDTH_32BIT:
6750	{
6751	uint32_t u32Val;
6752	rc = VMXReadVmcs32(uVmcsField, &u32Val);
6753	AssertRC(rc);
6754	Log4(("%-40s = %#RX32\n", s_aVmcsFields[i].pszName, u32Val));
6755	break;
6756	}
6757
6758	case VMX_VMCSFIELD_WIDTH_64BIT:
6759	case VMX_VMCSFIELD_WIDTH_NATURAL:
6760	{
6761	uint64_t u64Val;
6762	rc = VMXReadVmcs64(uVmcsField, &u64Val);
6763	AssertRC(rc);
6764	Log4(("%-40s = %#RX64\n", s_aVmcsFields[i].pszName, u64Val));
6765	break;
6766	}
6767	}
6768	}
6769	}
6770
6771	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6772	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6773	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6774	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6775	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6776	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6777	#endif /* VBOX_STRICT */
6778	break;
6779	}
6780
6781	default:
6782	/* Impossible */
6783	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6784	break;
6785	}
6786	}
6787
6788
6789	/**
6790	* Sets up the usage of TSC-offsetting and updates the VMCS.
6791	*
6792	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6793	* VMX-preemption timer.
6794	*
6795	* @returns VBox status code.
6796	* @param pVCpu The cross context virtual CPU structure.
6797	* @param pVmxTransient The VMX-transient structure.
6798	*
6799	* @remarks No-long-jump zone!!!
6800	*/
6801	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6802	{
6803	bool fOffsettedTsc;
6804	bool fParavirtTsc;
6805	uint64_t uTscOffset;
6806	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6807	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6808
6809	if (pVM->hm.s.vmx.fUsePreemptTimer)
6810	{
6811	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6812
6813	/* Make sure the returned values have sane upper and lower boundaries. */
6814	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6815	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6816	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6817	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6818
6819	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6820	* preemption timers here. We probably need to clamp the preemption timer,
6821	* after converting the timer value to the host. */
6822	uint32_t const cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6823	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6824	AssertRC(rc);
6825	}
6826	else
6827	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6828
6829	if (fParavirtTsc)
6830	{
6831	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6832	information before every VM-entry, hence disable it for performance sake. */
6833	#if 0
6834	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6835	AssertRC(rc);
6836	#endif
6837	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6838	}
6839
6840	if ( fOffsettedTsc
6841	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6842	{
6843	if (pVmxTransient->fIsNestedGuest)
6844	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6845	hmR0VmxSetTscOffsetVmcs(pVmcsInfo, uTscOffset);
6846	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6847	}
6848	else
6849	{
6850	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6851	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
6852	}
6853	}
6854
6855
6856	/**
6857	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6858	* VM-exit interruption info type.
6859	*
6860	* @returns The IEM exception flags.
6861	* @param uVector The event vector.
6862	* @param uVmxEventType The VMX event type.
6863	*
6864	* @remarks This function currently only constructs flags required for
6865	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6866	* and CR2 aspects of an exception are not included).
6867	*/
6868	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6869	{
6870	uint32_t fIemXcptFlags;
6871	switch (uVmxEventType)
6872	{
6873	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6874	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6875	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6876	break;
6877
6878	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6879	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6880	break;
6881
6882	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6883	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6884	break;
6885
6886	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6887	{
6888	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6889	if (uVector == X86_XCPT_BP)
6890	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6891	else if (uVector == X86_XCPT_OF)
6892	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6893	else
6894	{
6895	fIemXcptFlags = 0;
6896	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6897	}
6898	break;
6899	}
6900
6901	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6902	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6903	break;
6904
6905	default:
6906	fIemXcptFlags = 0;
6907	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6908	break;
6909	}
6910	return fIemXcptFlags;
6911	}
6912
6913
6914	/**
6915	* Sets an event as a pending event to be injected into the guest.
6916	*
6917	* @param pVCpu The cross context virtual CPU structure.
6918	* @param u32IntInfo The VM-entry interruption-information field.
6919	* @param cbInstr The VM-entry instruction length in bytes (for
6920	* software interrupts, exceptions and privileged
6921	* software exceptions).
6922	* @param u32ErrCode The VM-entry exception error code.
6923	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6924	* page-fault.
6925	*/
6926	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6927	RTGCUINTPTR GCPtrFaultAddress)
6928	{
6929	Assert(!pVCpu->hm.s.Event.fPending);
6930	pVCpu->hm.s.Event.fPending = true;
6931	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6932	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6933	pVCpu->hm.s.Event.cbInstr = cbInstr;
6934	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6935	}
6936
6937
6938	/**
6939	* Sets an external interrupt as pending-for-injection into the VM.
6940	*
6941	* @param pVCpu The cross context virtual CPU structure.
6942	* @param u8Interrupt The external interrupt vector.
6943	*/
6944	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
6945	{
6946	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6947	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6948	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6949	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6950	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6951	}
6952
6953
6954	/**
6955	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6956	*
6957	* @param pVCpu The cross context virtual CPU structure.
6958	*/
6959	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPUCC pVCpu)
6960	{
6961	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6962	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6963	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6964	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6965	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6966	}
6967
6968
6969	/**
6970	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6971	*
6972	* @param pVCpu The cross context virtual CPU structure.
6973	*/
6974	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPUCC pVCpu)
6975	{
6976	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6977	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6978	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6979	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6980	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6981	}
6982
6983
6984	/**
6985	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6986	*
6987	* @param pVCpu The cross context virtual CPU structure.
6988	*/
6989	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPUCC pVCpu)
6990	{
6991	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6992	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6993	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6994	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6995	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6996	}
6997
6998
6999	/**
7000	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7001	*
7002	* @param pVCpu The cross context virtual CPU structure.
7003	*/
7004	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPUCC pVCpu)
7005	{
7006	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7007	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7008	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7009	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7010	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7011	}
7012
7013
7014	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7015	/**
7016	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7017	*
7018	* @param pVCpu The cross context virtual CPU structure.
7019	* @param u32ErrCode The error code for the general-protection exception.
7020	*/
7021	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7022	{
7023	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7024	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7025	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7026	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7027	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7028	}
7029
7030
7031	/**
7032	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7033	*
7034	* @param pVCpu The cross context virtual CPU structure.
7035	* @param u32ErrCode The error code for the stack exception.
7036	*/
7037	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
7038	{
7039	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7040	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7041	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7042	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7043	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7044	}
7045	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7046
7047
7048	/**
7049	* Fixes up attributes for the specified segment register.
7050	*
7051	* @param pVCpu The cross context virtual CPU structure.
7052	* @param pSelReg The segment register that needs fixing.
7053	* @param idxSel The VMCS field for the corresponding segment register.
7054	*/
7055	static void hmR0VmxFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7056	{
7057	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7058
7059	/*
7060	* If VT-x marks the segment as unusable, most other bits remain undefined:
7061	* - For CS the L, D and G bits have meaning.
7062	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7063	* - For the remaining data segments no bits are defined.
7064	*
7065	* The present bit and the unusable bit has been observed to be set at the
7066	* same time (the selector was supposed to be invalid as we started executing
7067	* a V8086 interrupt in ring-0).
7068	*
7069	* What should be important for the rest of the VBox code, is that the P bit is
7070	* cleared. Some of the other VBox code recognizes the unusable bit, but
7071	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7072	* safe side here, we'll strip off P and other bits we don't care about. If
7073	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7074	*
7075	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7076	*/
7077	#ifdef VBOX_STRICT
7078	uint32_t const uAttr = pSelReg->Attr.u;
7079	#endif
7080
7081	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7082	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7083	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7084
7085	#ifdef VBOX_STRICT
7086	VMMRZCallRing3Disable(pVCpu);
7087	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7088	# ifdef DEBUG_bird
7089	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7090	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7091	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7092	# endif
7093	VMMRZCallRing3Enable(pVCpu);
7094	NOREF(uAttr);
7095	#endif
7096	RT_NOREF2(pVCpu, idxSel);
7097	}
7098
7099
7100	/**
7101	* Imports a guest segment register from the current VMCS into the guest-CPU
7102	* context.
7103	*
7104	* @param pVCpu The cross context virtual CPU structure.
7105	* @param iSegReg The segment register number (X86_SREG_XXX).
7106	*
7107	* @remarks Called with interrupts and/or preemption disabled.
7108	*/
7109	static void hmR0VmxImportGuestSegReg(PVMCPUCC pVCpu, uint8_t iSegReg)
7110	{
7111	Assert(iSegReg < X86_SREG_COUNT);
7112
7113	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7114	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7115	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7116	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7117
7118	uint16_t u16Sel;
7119	uint64_t u64Base;
7120	uint32_t u32Limit, u32Attr;
7121	int rc = VMXReadVmcs16(idxSel, &u16Sel); AssertRC(rc);
7122	rc = VMXReadVmcs32(idxLimit, &u32Limit); AssertRC(rc);
7123	rc = VMXReadVmcs32(idxAttr, &u32Attr); AssertRC(rc);
7124	rc = VMXReadVmcsNw(idxBase, &u64Base); AssertRC(rc);
7125
7126	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7127	pSelReg->Sel = u16Sel;
7128	pSelReg->ValidSel = u16Sel;
7129	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7130	pSelReg->u32Limit = u32Limit;
7131	pSelReg->u64Base = u64Base;
7132	pSelReg->Attr.u = u32Attr;
7133	if (u32Attr & X86DESCATTR_UNUSABLE)
7134	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7135	}
7136
7137
7138	/**
7139	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7140	*
7141	* @param pVCpu The cross context virtual CPU structure.
7142	*
7143	* @remarks Called with interrupts and/or preemption disabled.
7144	*/
7145	static void hmR0VmxImportGuestLdtr(PVMCPUCC pVCpu)
7146	{
7147	uint16_t u16Sel;
7148	uint64_t u64Base;
7149	uint32_t u32Limit, u32Attr;
7150	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
7151	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
7152	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7153	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
7154
7155	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
7156	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
7157	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7158	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7159	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7160	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7161	if (u32Attr & X86DESCATTR_UNUSABLE)
7162	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7163	}
7164
7165
7166	/**
7167	* Imports the guest TR from the current VMCS into the guest-CPU context.
7168	*
7169	* @param pVCpu The cross context virtual CPU structure.
7170	*
7171	* @remarks Called with interrupts and/or preemption disabled.
7172	*/
7173	static void hmR0VmxImportGuestTr(PVMCPUCC pVCpu)
7174	{
7175	uint16_t u16Sel;
7176	uint64_t u64Base;
7177	uint32_t u32Limit, u32Attr;
7178	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
7179	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
7180	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
7181	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
7182
7183	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
7184	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
7185	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7186	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7187	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7188	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7189	/* TR is the only selector that can never be unusable. */
7190	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7191	}
7192
7193
7194	/**
7195	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7196	*
7197	* @param pVCpu The cross context virtual CPU structure.
7198	*
7199	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7200	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7201	* instead!!!
7202	*/
7203	static void hmR0VmxImportGuestRip(PVMCPUCC pVCpu)
7204	{
7205	uint64_t u64Val;
7206	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7207	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7208	{
7209	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
7210	AssertRC(rc);
7211
7212	pCtx->rip = u64Val;
7213	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7214	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7215	}
7216	}
7217
7218
7219	/**
7220	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7221	*
7222	* @param pVCpu The cross context virtual CPU structure.
7223	* @param pVmcsInfo The VMCS info. object.
7224	*
7225	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7226	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7227	* instead!!!
7228	*/
7229	static void hmR0VmxImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7230	{
7231	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7232	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7233	{
7234	uint64_t u64Val;
7235	int rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7236	AssertRC(rc);
7237
7238	pCtx->rflags.u64 = u64Val;
7239	if (pVmcsInfo->RealMode.fRealOnV86Active)
7240	{
7241	pCtx->eflags.Bits.u1VM = 0;
7242	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7243	}
7244	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7245	}
7246	}
7247
7248
7249	/**
7250	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7251	* context.
7252	*
7253	* @param pVCpu The cross context virtual CPU structure.
7254	* @param pVmcsInfo The VMCS info. object.
7255	*
7256	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7257	* do not log!
7258	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7259	* instead!!!
7260	*/
7261	static void hmR0VmxImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
7262	{
7263	uint32_t u32Val;
7264	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
7265	if (!u32Val)
7266	{
7267	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7268	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7269	CPUMSetGuestNmiBlocking(pVCpu, false);
7270	}
7271	else
7272	{
7273	/*
7274	* We must import RIP here to set our EM interrupt-inhibited state.
7275	* We also import RFLAGS as our code that evaluates pending interrupts
7276	* before VM-entry requires it.
7277	*/
7278	hmR0VmxImportGuestRip(pVCpu);
7279	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7280
7281	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7282	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7283	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7284	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7285
7286	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7287	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7288	}
7289	}
7290
7291
7292	/**
7293	* Worker for VMXR0ImportStateOnDemand.
7294	*
7295	* @returns VBox status code.
7296	* @param pVCpu The cross context virtual CPU structure.
7297	* @param pVmcsInfo The VMCS info. object.
7298	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7299	*/
7300	static int hmR0VmxImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7301	{
7302	int rc = VINF_SUCCESS;
7303	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7304	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7305	uint32_t u32Val;
7306
7307	/*
7308	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7309	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7310	* neither are other host platforms.
7311	*
7312	* Committing this temporarily as it prevents BSOD.
7313	*
7314	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7315	*/
7316	#ifdef RT_OS_WINDOWS
7317	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7318	return VERR_HM_IPE_1;
7319	#endif
7320
7321	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7322
7323	/*
7324	* We disable interrupts to make the updating of the state and in particular
7325	* the fExtrn modification atomic wrt to preemption hooks.
7326	*/
7327	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7328
7329	fWhat &= pCtx->fExtrn;
7330	if (fWhat)
7331	{
7332	do
7333	{
7334	if (fWhat & CPUMCTX_EXTRN_RIP)
7335	hmR0VmxImportGuestRip(pVCpu);
7336
7337	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7338	hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7339
7340	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7341	hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7342
7343	if (fWhat & CPUMCTX_EXTRN_RSP)
7344	{
7345	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RSP, &pCtx->rsp);
7346	AssertRC(rc);
7347	}
7348
7349	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7350	{
7351	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7352	if (fWhat & CPUMCTX_EXTRN_CS)
7353	{
7354	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7355	hmR0VmxImportGuestRip(pVCpu);
7356	if (fRealOnV86Active)
7357	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7358	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7359	}
7360	if (fWhat & CPUMCTX_EXTRN_SS)
7361	{
7362	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7363	if (fRealOnV86Active)
7364	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7365	}
7366	if (fWhat & CPUMCTX_EXTRN_DS)
7367	{
7368	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7369	if (fRealOnV86Active)
7370	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7371	}
7372	if (fWhat & CPUMCTX_EXTRN_ES)
7373	{
7374	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7375	if (fRealOnV86Active)
7376	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7377	}
7378	if (fWhat & CPUMCTX_EXTRN_FS)
7379	{
7380	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7381	if (fRealOnV86Active)
7382	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7383	}
7384	if (fWhat & CPUMCTX_EXTRN_GS)
7385	{
7386	hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7387	if (fRealOnV86Active)
7388	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7389	}
7390	}
7391
7392	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7393	{
7394	if (fWhat & CPUMCTX_EXTRN_LDTR)
7395	hmR0VmxImportGuestLdtr(pVCpu);
7396
7397	if (fWhat & CPUMCTX_EXTRN_GDTR)
7398	{
7399	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
7400	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
7401	pCtx->gdtr.cbGdt = u32Val;
7402	}
7403
7404	/* Guest IDTR. */
7405	if (fWhat & CPUMCTX_EXTRN_IDTR)
7406	{
7407	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
7408	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
7409	pCtx->idtr.cbIdt = u32Val;
7410	}
7411
7412	/* Guest TR. */
7413	if (fWhat & CPUMCTX_EXTRN_TR)
7414	{
7415	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7416	don't need to import that one. */
7417	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7418	hmR0VmxImportGuestTr(pVCpu);
7419	}
7420	}
7421
7422	if (fWhat & CPUMCTX_EXTRN_DR7)
7423	{
7424	if (!pVCpu->hm.s.fUsingHyperDR7)
7425	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_DR7, &pCtx->dr[7]); AssertRC(rc);
7426	}
7427
7428	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7429	{
7430	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
7431	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
7432	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
7433	pCtx->SysEnter.cs = u32Val;
7434	}
7435
7436	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7437	{
7438	if ( pVM->hm.s.fAllow64BitGuests
7439	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7440	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7441	}
7442
7443	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7444	{
7445	if ( pVM->hm.s.fAllow64BitGuests
7446	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7447	{
7448	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7449	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7450	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7451	}
7452	}
7453
7454	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7455	{
7456	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7457	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7458	Assert(pMsrs);
7459	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7460	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7461	for (uint32_t i = 0; i < cMsrs; i++)
7462	{
7463	uint32_t const idMsr = pMsrs[i].u32Msr;
7464	switch (idMsr)
7465	{
7466	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7467	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7468	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7469	default:
7470	{
7471	pCtx->fExtrn = 0;
7472	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7473	ASMSetFlags(fEFlags);
7474	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7475	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7476	}
7477	}
7478	}
7479	}
7480
7481	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7482	{
7483	if (fWhat & CPUMCTX_EXTRN_CR0)
7484	{
7485	uint64_t u64Cr0;
7486	uint64_t u64Shadow;
7487	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
7488	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
7489	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7490	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7491	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7492	#else
7493	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7494	{
7495	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7496	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7497	}
7498	else
7499	{
7500	/*
7501	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
7502	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7503	* re-construct CR0. See @bugref{9180#c95} for details.
7504	*/
7505	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7506	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7507	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
7508	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
7509	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
7510	}
7511	#endif
7512	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7513	CPUMSetGuestCR0(pVCpu, u64Cr0);
7514	VMMRZCallRing3Enable(pVCpu);
7515	}
7516
7517	if (fWhat & CPUMCTX_EXTRN_CR4)
7518	{
7519	uint64_t u64Cr4;
7520	uint64_t u64Shadow;
7521	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
7522	rc \|= VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
7523	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
7524	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7525	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7526	#else
7527	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7528	{
7529	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7530	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7531	}
7532	else
7533	{
7534	/*
7535	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
7536	* the nested-guest using hardware-assisted VMX. Accordingly we need to
7537	* re-construct CR4. See @bugref{9180#c95} for details.
7538	*/
7539	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
7540	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7541	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
7542	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
7543	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
7544	}
7545	#endif
7546	pCtx->cr4 = u64Cr4;
7547	}
7548
7549	if (fWhat & CPUMCTX_EXTRN_CR3)
7550	{
7551	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7552	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7553	\|\| ( pVM->hm.s.fNestedPaging
7554	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7555	{
7556	uint64_t u64Cr3;
7557	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
7558	if (pCtx->cr3 != u64Cr3)
7559	{
7560	pCtx->cr3 = u64Cr3;
7561	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7562	}
7563
7564	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7565	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7566	if (CPUMIsGuestInPAEModeEx(pCtx))
7567	{
7568	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRC(rc);
7569	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRC(rc);
7570	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRC(rc);
7571	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRC(rc);
7572	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7573	}
7574	}
7575	}
7576	}
7577
7578	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7579	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7580	{
7581	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7582	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7583	{
7584	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7585	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7586	if (RT_SUCCESS(rc))
7587	{ /* likely */ }
7588	else
7589	break;
7590	}
7591	}
7592	#endif
7593	} while (0);
7594
7595	if (RT_SUCCESS(rc))
7596	{
7597	/* Update fExtrn. */
7598	pCtx->fExtrn &= ~fWhat;
7599
7600	/* If everything has been imported, clear the HM keeper bit. */
7601	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7602	{
7603	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7604	Assert(!pCtx->fExtrn);
7605	}
7606	}
7607	}
7608	else
7609	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7610
7611	/*
7612	* Restore interrupts.
7613	*/
7614	ASMSetFlags(fEFlags);
7615
7616	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7617
7618	if (RT_SUCCESS(rc))
7619	{ /* likely */ }
7620	else
7621	return rc;
7622
7623	/*
7624	* Honor any pending CR3 updates.
7625	*
7626	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
7627	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7628	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7629	*
7630	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7631	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7632	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7633	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7634	*
7635	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7636	*/
7637	if (VMMRZCallRing3IsEnabled(pVCpu))
7638	{
7639	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7640	{
7641	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7642	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7643	}
7644
7645	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7646	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7647
7648	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7649	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7650	}
7651
7652	return VINF_SUCCESS;
7653	}
7654
7655
7656	/**
7657	* Saves the guest state from the VMCS into the guest-CPU context.
7658	*
7659	* @returns VBox status code.
7660	* @param pVCpu The cross context virtual CPU structure.
7661	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7662	*/
7663	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
7664	{
7665	AssertPtr(pVCpu);
7666	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7667	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7668	}
7669
7670
7671	/**
7672	* Check per-VM and per-VCPU force flag actions that require us to go back to
7673	* ring-3 for one reason or another.
7674	*
7675	* @returns Strict VBox status code (i.e. informational status codes too)
7676	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7677	* ring-3.
7678	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7679	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7680	* interrupts)
7681	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7682	* all EMTs to be in ring-3.
7683	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7684	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7685	* to the EM loop.
7686	*
7687	* @param pVCpu The cross context virtual CPU structure.
7688	* @param fStepping Whether we are single-stepping the guest using the
7689	* hypervisor debugger.
7690	*
7691	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7692	* is no longer in VMX non-root mode.
7693	*/
7694	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPUCC pVCpu, bool fStepping)
7695	{
7696	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7697
7698	/*
7699	* Update pending interrupts into the APIC's IRR.
7700	*/
7701	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7702	APICUpdatePendingInterrupts(pVCpu);
7703
7704	/*
7705	* Anything pending? Should be more likely than not if we're doing a good job.
7706	*/
7707	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7708	if ( !fStepping
7709	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7710	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7711	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7712	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7713	return VINF_SUCCESS;
7714
7715	/* Pending PGM C3 sync. */
7716	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7717	{
7718	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7719	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7720	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7721	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7722	if (rcStrict != VINF_SUCCESS)
7723	{
7724	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7725	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7726	return rcStrict;
7727	}
7728	}
7729
7730	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7731	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7732	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7733	{
7734	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7735	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7736	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7737	return rc;
7738	}
7739
7740	/* Pending VM request packets, such as hardware interrupts. */
7741	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7742	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7743	{
7744	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7745	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7746	return VINF_EM_PENDING_REQUEST;
7747	}
7748
7749	/* Pending PGM pool flushes. */
7750	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7751	{
7752	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7753	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7754	return VINF_PGM_POOL_FLUSH_PENDING;
7755	}
7756
7757	/* Pending DMA requests. */
7758	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7759	{
7760	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7761	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7762	return VINF_EM_RAW_TO_R3;
7763	}
7764
7765	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7766	/* Pending nested-guest APIC-write (has highest priority among nested-guest FFs). */
7767	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7768	{
7769	Log4Func(("Pending nested-guest APIC-write\n"));
7770	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
7771	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7772	return rcStrict;
7773	}
7774	/** @todo VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_PREEMPT_TIMER */
7775	#endif
7776
7777	return VINF_SUCCESS;
7778	}
7779
7780
7781	/**
7782	* Converts any TRPM trap into a pending HM event. This is typically used when
7783	* entering from ring-3 (not longjmp returns).
7784	*
7785	* @param pVCpu The cross context virtual CPU structure.
7786	*/
7787	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
7788	{
7789	Assert(TRPMHasTrap(pVCpu));
7790	Assert(!pVCpu->hm.s.Event.fPending);
7791
7792	uint8_t uVector;
7793	TRPMEVENT enmTrpmEvent;
7794	RTGCUINT uErrCode;
7795	RTGCUINTPTR GCPtrFaultAddress;
7796	uint8_t cbInstr;
7797
7798	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7799	AssertRC(rc);
7800
7801	uint32_t u32IntInfo;
7802	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
7803	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent);
7804
7805	rc = TRPMResetTrap(pVCpu);
7806	AssertRC(rc);
7807	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7808	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7809
7810	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7811	}
7812
7813
7814	/**
7815	* Converts the pending HM event into a TRPM trap.
7816	*
7817	* @param pVCpu The cross context virtual CPU structure.
7818	*/
7819	static void hmR0VmxPendingEventToTrpmTrap(PVMCPUCC pVCpu)
7820	{
7821	Assert(pVCpu->hm.s.Event.fPending);
7822
7823	/* If a trap was already pending, we did something wrong! */
7824	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7825
7826	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
7827	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
7828	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
7829
7830	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7831
7832	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7833	AssertRC(rc);
7834
7835	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7836	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
7837
7838	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
7839	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7840	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
7841	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7842
7843	/* We're now done converting the pending event. */
7844	pVCpu->hm.s.Event.fPending = false;
7845	}
7846
7847
7848	/**
7849	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7850	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7851	*
7852	* @param pVCpu The cross context virtual CPU structure.
7853	* @param pVmcsInfo The VMCS info. object.
7854	*/
7855	static void hmR0VmxSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7856	{
7857	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7858	{
7859	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7860	{
7861	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7862	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7863	AssertRC(rc);
7864	}
7865	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7866	}
7867
7868
7869	/**
7870	* Clears the interrupt-window exiting control in the VMCS.
7871	*
7872	* @param pVmcsInfo The VMCS info. object.
7873	*/
7874	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7875	{
7876	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7877	{
7878	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7879	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7880	AssertRC(rc);
7881	}
7882	}
7883
7884
7885	/**
7886	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7887	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7888	*
7889	* @param pVCpu The cross context virtual CPU structure.
7890	* @param pVmcsInfo The VMCS info. object.
7891	*/
7892	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
7893	{
7894	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7895	{
7896	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7897	{
7898	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7899	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7900	AssertRC(rc);
7901	Log4Func(("Setup NMI-window exiting\n"));
7902	}
7903	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7904	}
7905
7906
7907	/**
7908	* Clears the NMI-window exiting control in the VMCS.
7909	*
7910	* @param pVmcsInfo The VMCS info. object.
7911	*/
7912	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7913	{
7914	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7915	{
7916	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7917	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7918	AssertRC(rc);
7919	}
7920	}
7921
7922
7923	/**
7924	* Does the necessary state syncing before returning to ring-3 for any reason
7925	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7926	*
7927	* @returns VBox status code.
7928	* @param pVCpu The cross context virtual CPU structure.
7929	* @param fImportState Whether to import the guest state from the VMCS back
7930	* to the guest-CPU context.
7931	*
7932	* @remarks No-long-jmp zone!!!
7933	*/
7934	static int hmR0VmxLeave(PVMCPUCC pVCpu, bool fImportState)
7935	{
7936	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7937	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7938
7939	RTCPUID const idCpu = RTMpCpuId();
7940	Log4Func(("HostCpuId=%u\n", idCpu));
7941
7942	/*
7943	* !!! IMPORTANT !!!
7944	* If you modify code here, check whether VMXR0CallRing3Callback() needs to be updated too.
7945	*/
7946
7947	/* Save the guest state if necessary. */
7948	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7949	if (fImportState)
7950	{
7951	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
7952	AssertRCReturn(rc, rc);
7953	}
7954
7955	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7956	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7957	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7958
7959	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7960	#ifdef VBOX_STRICT
7961	if (CPUMIsHyperDebugStateActive(pVCpu))
7962	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7963	#endif
7964	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7965	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7966	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7967
7968	/* Restore host-state bits that VT-x only restores partially. */
7969	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7970	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7971	{
7972	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7973	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7974	}
7975	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7976
7977	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7978	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7979	{
7980	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7981	if (!fImportState)
7982	{
7983	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
7984	AssertRCReturn(rc, rc);
7985	}
7986	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7987	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7988	}
7989	else
7990	pVCpu->hm.s.vmx.fLazyMsrs = 0;
7991
7992	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7993	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
7994
7995	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7996	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7997	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7998	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7999	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8000	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8001	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8002	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8003	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8004	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8005
8006	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8007
8008	/** @todo This partially defeats the purpose of having preemption hooks.
8009	* The problem is, deregistering the hooks should be moved to a place that
8010	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8011	* context.
8012	*/
8013	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8014	AssertRCReturn(rc, rc);
8015
8016	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8017	/*
8018	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8019	* clear a shadow VMCS before allowing that VMCS to become active on another
8020	* logical processor. We may or may not be importing guest state which clears
8021	* it, so cover for it here.
8022	*
8023	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8024	*/
8025	if ( pVmcsInfo->pvShadowVmcs
8026	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8027	{
8028	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8029	AssertRCReturn(rc, rc);
8030	}
8031
8032	/*
8033	* Flag that we need to re-export the host state if we switch to this VMCS before
8034	* executing guest or nested-guest code.
8035	*/
8036	pVmcsInfo->idHostCpuState = NIL_RTCPUID;
8037	#endif
8038
8039	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8040	NOREF(idCpu);
8041	return VINF_SUCCESS;
8042	}
8043
8044
8045	/**
8046	* Leaves the VT-x session.
8047	*
8048	* @returns VBox status code.
8049	* @param pVCpu The cross context virtual CPU structure.
8050	*
8051	* @remarks No-long-jmp zone!!!
8052	*/
8053	static int hmR0VmxLeaveSession(PVMCPUCC pVCpu)
8054	{
8055	HM_DISABLE_PREEMPT(pVCpu);
8056	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8057	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8058	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8059
8060	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8061	and done this from the VMXR0ThreadCtxCallback(). */
8062	if (!pVCpu->hm.s.fLeaveDone)
8063	{
8064	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8065	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8066	pVCpu->hm.s.fLeaveDone = true;
8067	}
8068	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8069
8070	/*
8071	* !!! IMPORTANT !!!
8072	* If you modify code here, make sure to check whether VMXR0CallRing3Callback() needs to be updated too.
8073	*/
8074
8075	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8076	/** @todo Deregistering here means we need to VMCLEAR always
8077	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8078	* for calling VMMR0ThreadCtxHookDisable here! */
8079	VMMR0ThreadCtxHookDisable(pVCpu);
8080
8081	/* Leave HM context. This takes care of local init (term) and deregistering the longjmp-to-ring-3 callback. */
8082	int rc = HMR0LeaveCpu(pVCpu);
8083	HM_RESTORE_PREEMPT();
8084	return rc;
8085	}
8086
8087
8088	/**
8089	* Does the necessary state syncing before doing a longjmp to ring-3.
8090	*
8091	* @returns VBox status code.
8092	* @param pVCpu The cross context virtual CPU structure.
8093	*
8094	* @remarks No-long-jmp zone!!!
8095	*/
8096	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPUCC pVCpu)
8097	{
8098	return hmR0VmxLeaveSession(pVCpu);
8099	}
8100
8101
8102	/**
8103	* Take necessary actions before going back to ring-3.
8104	*
8105	* An action requires us to go back to ring-3. This function does the necessary
8106	* steps before we can safely return to ring-3. This is not the same as longjmps
8107	* to ring-3, this is voluntary and prepares the guest so it may continue
8108	* executing outside HM (recompiler/IEM).
8109	*
8110	* @returns VBox status code.
8111	* @param pVCpu The cross context virtual CPU structure.
8112	* @param rcExit The reason for exiting to ring-3. Can be
8113	* VINF_VMM_UNKNOWN_RING3_CALL.
8114	*/
8115	static int hmR0VmxExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
8116	{
8117	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8118
8119	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8120	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8121	{
8122	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8123	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8124	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8125	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8126	}
8127
8128	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8129	VMMRZCallRing3Disable(pVCpu);
8130	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8131
8132	/*
8133	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8134	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8135	*
8136	* This is because execution may continue from ring-3 and we would need to inject
8137	* the event from there (hence place it back in TRPM).
8138	*/
8139	if (pVCpu->hm.s.Event.fPending)
8140	{
8141	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8142	Assert(!pVCpu->hm.s.Event.fPending);
8143
8144	/* Clear the events from the VMCS. */
8145	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8146	AssertRC(rc);
8147	}
8148	#ifdef VBOX_STRICT
8149	else
8150	{
8151	/*
8152	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8153	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8154	* occasionally, see @bugref{9180#c42}.
8155	*
8156	* However, if the VM-entry failed, any VM entry-interruption info. field would
8157	* be left unmodified as the event would not have been injected to the guest. In
8158	* such cases, don't assert, we're not going to continue guest execution anyway.
8159	*/
8160	uint32_t uExitReason;
8161	uint32_t uEntryIntInfo;
8162	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8163	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8164	AssertRC(rc);
8165	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8166	}
8167	#endif
8168
8169	/*
8170	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8171	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8172	* (e.g. TPR below threshold).
8173	*/
8174	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8175	{
8176	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8177	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8178	}
8179
8180	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8181	and if we're injecting an event we should have a TRPM trap pending. */
8182	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8183	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8184	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8185	#endif
8186
8187	/* Save guest state and restore host state bits. */
8188	int rc = hmR0VmxLeaveSession(pVCpu);
8189	AssertRCReturn(rc, rc);
8190	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8191
8192	/* Thread-context hooks are unregistered at this point!!! */
8193	/* Ring-3 callback notifications are unregistered at this point!!! */
8194
8195	/* Sync recompiler state. */
8196	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8197	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8198	\| CPUM_CHANGED_LDTR
8199	\| CPUM_CHANGED_GDTR
8200	\| CPUM_CHANGED_IDTR
8201	\| CPUM_CHANGED_TR
8202	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8203	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8204	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8205	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8206
8207	Assert(!pVCpu->hm.s.fClearTrapFlag);
8208
8209	/* Update the exit-to-ring 3 reason. */
8210	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8211
8212	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8213	if ( rcExit != VINF_EM_RAW_INTERRUPT
8214	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8215	{
8216	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8217	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8218	}
8219
8220	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8221	VMMRZCallRing3Enable(pVCpu);
8222	return rc;
8223	}
8224
8225
8226	/**
8227	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8228	* longjump to ring-3 and possibly get preempted.
8229	*
8230	* @returns VBox status code.
8231	* @param pVCpu The cross context virtual CPU structure.
8232	* @param enmOperation The operation causing the ring-3 longjump.
8233	*/
8234	VMMR0DECL(int) VMXR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
8235	{
8236	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8237	{
8238	/*
8239	* !!! IMPORTANT !!!
8240	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8241	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8242	*/
8243	VMMRZCallRing3RemoveNotification(pVCpu);
8244	VMMRZCallRing3Disable(pVCpu);
8245	HM_DISABLE_PREEMPT(pVCpu);
8246
8247	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8248	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8249	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8250	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8251
8252	/* Restore host-state bits that VT-x only restores partially. */
8253	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8254	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8255	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8256	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8257
8258	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8259	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8260	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8261
8262	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8263	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8264	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8265
8266	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8267	cleared as part of importing the guest state above. */
8268	hmR0VmxClearVmcs(pVmcsInfo);
8269
8270	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8271	VMMR0ThreadCtxHookDisable(pVCpu);
8272
8273	/* Leave HM context. This takes care of local init (term). */
8274	HMR0LeaveCpu(pVCpu);
8275	HM_RESTORE_PREEMPT();
8276	return VINF_SUCCESS;
8277	}
8278
8279	Assert(pVCpu);
8280	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8281	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8282
8283	VMMRZCallRing3Disable(pVCpu);
8284	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8285
8286	Log4Func(("-> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8287
8288	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8289	AssertRCReturn(rc, rc);
8290
8291	VMMRZCallRing3Enable(pVCpu);
8292	return VINF_SUCCESS;
8293	}
8294
8295
8296	/**
8297	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8298	* stack.
8299	*
8300	* @returns Strict VBox status code (i.e. informational status codes too).
8301	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8302	* @param pVCpu The cross context virtual CPU structure.
8303	* @param uValue The value to push to the guest stack.
8304	*/
8305	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPUCC pVCpu, uint16_t uValue)
8306	{
8307	/*
8308	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8309	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8310	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8311	*/
8312	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8313	if (pCtx->sp == 1)
8314	return VINF_EM_RESET;
8315	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8316	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8317	AssertRC(rc);
8318	return rc;
8319	}
8320
8321
8322	/**
8323	* Injects an event into the guest upon VM-entry by updating the relevant fields
8324	* in the VM-entry area in the VMCS.
8325	*
8326	* @returns Strict VBox status code (i.e. informational status codes too).
8327	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8328	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8329	*
8330	* @param pVCpu The cross context virtual CPU structure.
8331	* @param pVmxTransient The VMX-transient structure.
8332	* @param pEvent The event being injected.
8333	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8334	* will be updated if necessary. This cannot not be NULL.
8335	* @param fStepping Whether we're single-stepping guest execution and should
8336	* return VINF_EM_DBG_STEPPED if the event is injected
8337	* directly (registers modified by us, not by hardware on
8338	* VM-entry).
8339	*/
8340	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8341	uint32_t *pfIntrState)
8342	{
8343	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8344	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8345	Assert(pfIntrState);
8346
8347	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8348	uint32_t u32IntInfo = pEvent->u64IntInfo;
8349	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8350	uint32_t const cbInstr = pEvent->cbInstr;
8351	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8352	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8353	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8354
8355	#ifdef VBOX_STRICT
8356	/*
8357	* Validate the error-code-valid bit for hardware exceptions.
8358	* No error codes for exceptions in real-mode.
8359	*
8360	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8361	*/
8362	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8363	&& !CPUMIsGuestInRealModeEx(pCtx))
8364	{
8365	switch (uVector)
8366	{
8367	case X86_XCPT_PF:
8368	case X86_XCPT_DF:
8369	case X86_XCPT_TS:
8370	case X86_XCPT_NP:
8371	case X86_XCPT_SS:
8372	case X86_XCPT_GP:
8373	case X86_XCPT_AC:
8374	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8375	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8376	RT_FALL_THRU();
8377	default:
8378	break;
8379	}
8380	}
8381
8382	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8383	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8384	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8385	#endif
8386
8387	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8388
8389	/*
8390	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8391	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8392	* interrupt handler in the (real-mode) guest.
8393	*
8394	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8395	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8396	*/
8397	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8398	{
8399	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8400	{
8401	/*
8402	* For CPUs with unrestricted guest execution enabled and with the guest
8403	* in real-mode, we must not set the deliver-error-code bit.
8404	*
8405	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8406	*/
8407	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8408	}
8409	else
8410	{
8411	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8412	Assert(PDMVmmDevHeapIsEnabled(pVM));
8413	Assert(pVM->hm.s.vmx.pRealModeTSS);
8414	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8415
8416	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8417	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8418	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8419	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8420	AssertRCReturn(rc2, rc2);
8421
8422	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8423	size_t const cbIdtEntry = sizeof(X86IDTR16);
8424	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8425	{
8426	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8427	if (uVector == X86_XCPT_DF)
8428	return VINF_EM_RESET;
8429
8430	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8431	No error codes for exceptions in real-mode. */
8432	if (uVector == X86_XCPT_GP)
8433	{
8434	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8435	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8436	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8437	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8438	HMEVENT EventXcptDf;
8439	RT_ZERO(EventXcptDf);
8440	EventXcptDf.u64IntInfo = uXcptDfInfo;
8441	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8442	}
8443
8444	/*
8445	* If we're injecting an event with no valid IDT entry, inject a #GP.
8446	* No error codes for exceptions in real-mode.
8447	*
8448	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8449	*/
8450	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8451	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8452	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8453	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8454	HMEVENT EventXcptGp;
8455	RT_ZERO(EventXcptGp);
8456	EventXcptGp.u64IntInfo = uXcptGpInfo;
8457	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8458	}
8459
8460	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8461	uint16_t uGuestIp = pCtx->ip;
8462	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8463	{
8464	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8465	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8466	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8467	}
8468	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8469	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8470
8471	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8472	X86IDTR16 IdtEntry;
8473	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8474	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8475	AssertRCReturn(rc2, rc2);
8476
8477	/* Construct the stack frame for the interrupt/exception handler. */
8478	VBOXSTRICTRC rcStrict;
8479	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8480	if (rcStrict == VINF_SUCCESS)
8481	{
8482	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8483	if (rcStrict == VINF_SUCCESS)
8484	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8485	}
8486
8487	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8488	if (rcStrict == VINF_SUCCESS)
8489	{
8490	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8491	pCtx->rip = IdtEntry.offSel;
8492	pCtx->cs.Sel = IdtEntry.uSel;
8493	pCtx->cs.ValidSel = IdtEntry.uSel;
8494	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8495	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8496	&& uVector == X86_XCPT_PF)
8497	pCtx->cr2 = GCPtrFault;
8498
8499	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8500	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8501	\| HM_CHANGED_GUEST_RSP);
8502
8503	/*
8504	* If we delivered a hardware exception (other than an NMI) and if there was
8505	* block-by-STI in effect, we should clear it.
8506	*/
8507	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8508	{
8509	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8510	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8511	Log4Func(("Clearing inhibition due to STI\n"));
8512	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8513	}
8514
8515	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8516	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8517
8518	/*
8519	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8520	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8521	*/
8522	pVCpu->hm.s.Event.fPending = false;
8523
8524	/*
8525	* If we eventually support nested-guest execution without unrestricted guest execution,
8526	* we should set fInterceptEvents here.
8527	*/
8528	Assert(!pVmxTransient->fIsNestedGuest);
8529
8530	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8531	if (fStepping)
8532	rcStrict = VINF_EM_DBG_STEPPED;
8533	}
8534	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8535	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8536	return rcStrict;
8537	}
8538	}
8539
8540	/*
8541	* Validate.
8542	*/
8543	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8544	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8545
8546	/*
8547	* Inject the event into the VMCS.
8548	*/
8549	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8550	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8551	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8552	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8553	AssertRC(rc);
8554
8555	/*
8556	* Update guest CR2 if this is a page-fault.
8557	*/
8558	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8559	pCtx->cr2 = GCPtrFault;
8560
8561	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8562	return VINF_SUCCESS;
8563	}
8564
8565
8566	/**
8567	* Evaluates the event to be delivered to the guest and sets it as the pending
8568	* event.
8569	*
8570	* @returns Strict VBox status code (i.e. informational status codes too).
8571	* @param pVCpu The cross context virtual CPU structure.
8572	* @param pVmxTransient The VMX-transient structure.
8573	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8574	*/
8575	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8576	{
8577	Assert(pfIntrState);
8578	Assert(!TRPMHasTrap(pVCpu));
8579
8580	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8581	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8582	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8583
8584	/*
8585	* Get the current interruptibility-state of the guest or nested-guest and
8586	* then figure out what needs to be injected.
8587	*/
8588	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8589	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8590	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8591	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8592
8593	/* We don't support block-by-SMI yet.*/
8594	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
8595
8596	/* Block-by-STI must not be set when interrupts are disabled. */
8597	if (fBlockSti)
8598	{
8599	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
8600	Assert(pCtx->eflags.Bits.u1IF);
8601	}
8602
8603	/* Update interruptibility state to the caller. */
8604	*pfIntrState = fIntrState;
8605
8606	/*
8607	* Toggling of interrupt force-flags here is safe since we update TRPM on
8608	* premature exits to ring-3 before executing guest code, see hmR0VmxExitToRing3().
8609	* We must NOT restore these force-flags.
8610	*/
8611
8612	/** @todo SMI. SMIs take priority over NMIs. */
8613
8614	/*
8615	* Check if an NMI is pending and if the guest or nested-guest can receive them.
8616	* NMIs take priority over external interrupts.
8617	*/
8618	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8619	{
8620	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8621	if ( !pVCpu->hm.s.Event.fPending
8622	&& !fBlockNmi
8623	&& !fBlockSti
8624	&& !fBlockMovSS)
8625	{
8626	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8627	if ( fIsNestedGuest
8628	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8629	return IEMExecVmxVmexitXcptNmi(pVCpu);
8630	#endif
8631	hmR0VmxSetPendingXcptNmi(pVCpu);
8632	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8633	Log4Func(("Pending NMI\n"));
8634	}
8635	else if (!fIsNestedGuest)
8636	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8637	/* else: for nested-guests, NMI-window exiting will be picked up when merging VMCS controls. */
8638	}
8639	/*
8640	* Check if an external interrupt (PIC/APIC) is pending and if the guest or nested-guest
8641	* can receive them. Once PDMGetInterrupt() returns a valid interrupt we -must- deliver
8642	* the interrupt. We can no longer re-request it from the APIC.
8643	*/
8644	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8645	&& !pVCpu->hm.s.fSingleInstruction)
8646	{
8647	Assert(!DBGFIsStepping(pVCpu));
8648	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8649	AssertRCReturn(rc, rc);
8650
8651	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8652	if ( !pVCpu->hm.s.Event.fPending
8653	&& !fBlockInt
8654	&& !fBlockSti
8655	&& !fBlockMovSS)
8656	{
8657	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8658	if ( fIsNestedGuest
8659	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8660	&& !CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8661	{
8662	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8663	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8664	return rcStrict;
8665	}
8666	#endif
8667	uint8_t u8Interrupt;
8668	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8669	if (RT_SUCCESS(rc))
8670	{
8671	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8672	if ( fIsNestedGuest
8673	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8674	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8675	{
8676	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8677	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8678	return rcStrict;
8679	}
8680	#endif
8681	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8682	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8683	}
8684	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8685	{
8686	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8687
8688	if ( !fIsNestedGuest
8689	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8690	hmR0VmxApicSetTprThreshold(pVmcsInfo, u8Interrupt >> 4);
8691	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8692
8693	/*
8694	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8695	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8696	* need to re-set this force-flag here.
8697	*/
8698	}
8699	else
8700	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8701	}
8702	else if (!fIsNestedGuest)
8703	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8704	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCS controls. */
8705	}
8706
8707	return VINF_SUCCESS;
8708	}
8709
8710
8711	/**
8712	* Injects any pending events into the guest if the guest is in a state to
8713	* receive them.
8714	*
8715	* @returns Strict VBox status code (i.e. informational status codes too).
8716	* @param pVCpu The cross context virtual CPU structure.
8717	* @param pVmxTransient The VMX-transient structure.
8718	* @param fIntrState The VT-x guest-interruptibility state.
8719	* @param fStepping Whether we are single-stepping the guest using the
8720	* hypervisor debugger and should return
8721	* VINF_EM_DBG_STEPPED if the event was dispatched
8722	* directly.
8723	*/
8724	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8725	{
8726	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8727	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8728
8729	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8730	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8731
8732	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8733	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8734	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8735	Assert(!TRPMHasTrap(pVCpu));
8736
8737	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8738	if (pVCpu->hm.s.Event.fPending)
8739	{
8740	/*
8741	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8742	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8743	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8744	*
8745	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8746	*/
8747	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8748	#ifdef VBOX_STRICT
8749	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8750	{
8751	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8752	Assert(!fBlockInt);
8753	Assert(!fBlockSti);
8754	Assert(!fBlockMovSS);
8755	}
8756	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8757	{
8758	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8759	Assert(!fBlockSti);
8760	Assert(!fBlockMovSS);
8761	Assert(!fBlockNmi);
8762	}
8763	#endif
8764	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8765	uIntType));
8766
8767	/*
8768	* Inject the event and get any changes to the guest-interruptibility state.
8769	*
8770	* The guest-interruptibility state may need to be updated if we inject the event
8771	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8772	*/
8773	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8774	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8775
8776	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8777	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8778	else
8779	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8780	}
8781
8782	/*
8783	* Update the guest-interruptibility state.
8784	*
8785	* This is required for the real-on-v86 software interrupt injection case above, as well as
8786	* updates to the guest state from ring-3 or IEM/REM.
8787	*/
8788	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8789	AssertRC(rc);
8790
8791	/*
8792	* There's no need to clear the VM-entry interruption-information field here if we're not
8793	* injecting anything. VT-x clears the valid bit on every VM-exit.
8794	*
8795	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8796	*/
8797
8798	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8799	NOREF(fBlockMovSS); NOREF(fBlockSti);
8800	return rcStrict;
8801	}
8802
8803
8804	/**
8805	* Enters the VT-x session.
8806	*
8807	* @returns VBox status code.
8808	* @param pVCpu The cross context virtual CPU structure.
8809	*/
8810	VMMR0DECL(int) VMXR0Enter(PVMCPUCC pVCpu)
8811	{
8812	AssertPtr(pVCpu);
8813	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8814	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8815
8816	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8817	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8818	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8819
8820	#ifdef VBOX_STRICT
8821	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8822	RTCCUINTREG uHostCr4 = ASMGetCR4();
8823	if (!(uHostCr4 & X86_CR4_VMXE))
8824	{
8825	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8826	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8827	}
8828	#endif
8829
8830	/*
8831	* Load the appropriate VMCS as the current and active one.
8832	*/
8833	PVMXVMCSINFO pVmcsInfo;
8834	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8835	if (!fInNestedGuestMode)
8836	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8837	else
8838	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8839	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8840	if (RT_SUCCESS(rc))
8841	{
8842	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8843	pVCpu->hm.s.fLeaveDone = false;
8844	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8845
8846	/*
8847	* Do the EMT scheduled L1D flush here if needed.
8848	*/
8849	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8850	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8851	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8852	hmR0MdsClear();
8853	}
8854	return rc;
8855	}
8856
8857
8858	/**
8859	* The thread-context callback (only on platforms which support it).
8860	*
8861	* @param enmEvent The thread-context event.
8862	* @param pVCpu The cross context virtual CPU structure.
8863	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8864	* @thread EMT(pVCpu)
8865	*/
8866	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
8867	{
8868	AssertPtr(pVCpu);
8869	RT_NOREF1(fGlobalInit);
8870
8871	switch (enmEvent)
8872	{
8873	case RTTHREADCTXEVENT_OUT:
8874	{
8875	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8876	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8877	VMCPU_ASSERT_EMT(pVCpu);
8878
8879	/* No longjmps (logger flushes, locks) in this fragile context. */
8880	VMMRZCallRing3Disable(pVCpu);
8881	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8882
8883	/* Restore host-state (FPU, debug etc.) */
8884	if (!pVCpu->hm.s.fLeaveDone)
8885	{
8886	/*
8887	* Do -not- import the guest-state here as we might already be in the middle of importing
8888	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8889	*/
8890	hmR0VmxLeave(pVCpu, false /* fImportState */);
8891	pVCpu->hm.s.fLeaveDone = true;
8892	}
8893
8894	/* Leave HM context, takes care of local init (term). */
8895	int rc = HMR0LeaveCpu(pVCpu);
8896	AssertRC(rc);
8897
8898	/* Restore longjmp state. */
8899	VMMRZCallRing3Enable(pVCpu);
8900	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8901	break;
8902	}
8903
8904	case RTTHREADCTXEVENT_IN:
8905	{
8906	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8907	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8908	VMCPU_ASSERT_EMT(pVCpu);
8909
8910	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8911	VMMRZCallRing3Disable(pVCpu);
8912	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8913
8914	/* Initialize the bare minimum state required for HM. This takes care of
8915	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8916	int rc = hmR0EnterCpu(pVCpu);
8917	AssertRC(rc);
8918	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8919	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8920
8921	/* Load the active VMCS as the current one. */
8922	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8923	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8924	AssertRC(rc);
8925	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8926	pVCpu->hm.s.fLeaveDone = false;
8927
8928	/* Do the EMT scheduled L1D flush if needed. */
8929	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8930	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8931
8932	/* Restore longjmp state. */
8933	VMMRZCallRing3Enable(pVCpu);
8934	break;
8935	}
8936
8937	default:
8938	break;
8939	}
8940	}
8941
8942
8943	/**
8944	* Exports the host state into the VMCS host-state area.
8945	* Sets up the VM-exit MSR-load area.
8946	*
8947	* The CPU state will be loaded from these fields on every successful VM-exit.
8948	*
8949	* @returns VBox status code.
8950	* @param pVCpu The cross context virtual CPU structure.
8951	*
8952	* @remarks No-long-jump zone!!!
8953	*/
8954	static int hmR0VmxExportHostState(PVMCPUCC pVCpu)
8955	{
8956	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8957
8958	int rc = VINF_SUCCESS;
8959	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8960	{
8961	hmR0VmxExportHostControlRegs();
8962
8963	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8964	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8965
8966	hmR0VmxExportHostMsrs(pVCpu);
8967
8968	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8969	}
8970	return rc;
8971	}
8972
8973
8974	/**
8975	* Saves the host state in the VMCS host-state.
8976	*
8977	* @returns VBox status code.
8978	* @param pVCpu The cross context virtual CPU structure.
8979	*
8980	* @remarks No-long-jump zone!!!
8981	*/
8982	VMMR0DECL(int) VMXR0ExportHostState(PVMCPUCC pVCpu)
8983	{
8984	AssertPtr(pVCpu);
8985	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8986
8987	/*
8988	* Export the host state here while entering HM context.
8989	* When thread-context hooks are used, we might get preempted and have to re-save the host
8990	* state but most of the time we won't be, so do it here before we disable interrupts.
8991	*/
8992	return hmR0VmxExportHostState(pVCpu);
8993	}
8994
8995
8996	/**
8997	* Exports the guest state into the VMCS guest-state area.
8998	*
8999	* The will typically be done before VM-entry when the guest-CPU state and the
9000	* VMCS state may potentially be out of sync.
9001	*
9002	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9003	* VM-entry controls.
9004	* Sets up the appropriate VMX non-root function to execute guest code based on
9005	* the guest CPU mode.
9006	*
9007	* @returns VBox strict status code.
9008	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9009	* without unrestricted guest execution and the VMMDev is not presently
9010	* mapped (e.g. EFI32).
9011	*
9012	* @param pVCpu The cross context virtual CPU structure.
9013	* @param pVmxTransient The VMX-transient structure.
9014	*
9015	* @remarks No-long-jump zone!!!
9016	*/
9017	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9018	{
9019	AssertPtr(pVCpu);
9020	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9021	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9022
9023	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9024
9025	/*
9026	* Determine real-on-v86 mode.
9027	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9028	*/
9029	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9030	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9031	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9032	pVmcsInfo->RealMode. fRealOnV86Active = false;
9033	else
9034	{
9035	Assert(!pVmxTransient->fIsNestedGuest);
9036	pVmcsInfo->RealMode.fRealOnV86Active = true;
9037	}
9038
9039	/*
9040	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9041	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9042	*/
9043	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9044	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9045	* be a need to evaluate this everytime since I'm pretty sure we intercept
9046	* all guest paging mode changes. */
9047	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9048	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9049
9050	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9051	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9052
9053	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9054	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9055
9056	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9057	if (rcStrict == VINF_SUCCESS)
9058	{ /* likely */ }
9059	else
9060	{
9061	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9062	return rcStrict;
9063	}
9064
9065	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9066	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9067
9068	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9069	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9070
9071	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9072	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9073
9074	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9075	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9076
9077	rc = hmR0VmxExportGuestRip(pVCpu);
9078	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9079	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9080	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9081
9082	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9083	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9084
9085	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9086	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9087	\| HM_CHANGED_GUEST_CR2
9088	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9089	\| HM_CHANGED_GUEST_X87
9090	\| HM_CHANGED_GUEST_SSE_AVX
9091	\| HM_CHANGED_GUEST_OTHER_XSAVE
9092	\| HM_CHANGED_GUEST_XCRx
9093	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9094	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9095	\| HM_CHANGED_GUEST_TSC_AUX
9096	\| HM_CHANGED_GUEST_OTHER_MSRS
9097	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9098
9099	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9100	return rc;
9101	}
9102
9103
9104	/**
9105	* Exports the state shared between the host and guest into the VMCS.
9106	*
9107	* @param pVCpu The cross context virtual CPU structure.
9108	* @param pVmxTransient The VMX-transient structure.
9109	*
9110	* @remarks No-long-jump zone!!!
9111	*/
9112	static void hmR0VmxExportSharedState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9113	{
9114	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9115	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9116
9117	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9118	{
9119	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9120	AssertRC(rc);
9121	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9122
9123	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9124	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9125	{
9126	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9127	AssertRC(rc);
9128	}
9129	}
9130
9131	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9132	{
9133	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9134	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9135	}
9136
9137	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9138	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9139	}
9140
9141
9142	/**
9143	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9144	*
9145	* @returns Strict VBox status code (i.e. informational status codes too).
9146	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9147	* without unrestricted guest execution and the VMMDev is not presently
9148	* mapped (e.g. EFI32).
9149	*
9150	* @param pVCpu The cross context virtual CPU structure.
9151	* @param pVmxTransient The VMX-transient structure.
9152	*
9153	* @remarks No-long-jump zone!!!
9154	*/
9155	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9156	{
9157	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9158	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9159	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9160
9161	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9162	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9163	#endif
9164
9165	/*
9166	* For many exits it's only RIP that changes and hence try to export it first
9167	* without going through a lot of change flag checks.
9168	*/
9169	VBOXSTRICTRC rcStrict;
9170	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9171	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9172	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9173	{
9174	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9175	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9176	{ /* likely */}
9177	else
9178	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9179	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9180	}
9181	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9182	{
9183	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9184	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9185	{ /* likely */}
9186	else
9187	{
9188	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9189	VBOXSTRICTRC_VAL(rcStrict)));
9190	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9191	return rcStrict;
9192	}
9193	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9194	}
9195	else
9196	rcStrict = VINF_SUCCESS;
9197
9198	#ifdef VBOX_STRICT
9199	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9200	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9201	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9202	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9203	("fCtxChanged=%#RX64\n", fCtxChanged));
9204	#endif
9205	return rcStrict;
9206	}
9207
9208
9209	/**
9210	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9211	* and update error record fields accordingly.
9212	*
9213	* @returns VMX_IGS_* error codes.
9214	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9215	* wrong with the guest state.
9216	*
9217	* @param pVCpu The cross context virtual CPU structure.
9218	* @param pVmcsInfo The VMCS info. object.
9219	*
9220	* @remarks This function assumes our cache of the VMCS controls
9221	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9222	*/
9223	static uint32_t hmR0VmxCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9224	{
9225	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9226	#define HMVMX_CHECK_BREAK(expr, err) do { \
9227	if (!(expr)) { uError = (err); break; } \
9228	} while (0)
9229
9230	int rc;
9231	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9232	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9233	uint32_t uError = VMX_IGS_ERROR;
9234	uint32_t u32Val;
9235	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9236
9237	do
9238	{
9239	/*
9240	* CR0.
9241	*/
9242	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9243	uint64_t fSetCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9244	uint64_t const fZapCr0 = (pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9245	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
9246	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9247	if (fUnrestrictedGuest)
9248	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
9249
9250	uint64_t u64GuestCr0;
9251	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64GuestCr0);
9252	AssertRC(rc);
9253	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9254	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9255	if ( !fUnrestrictedGuest
9256	&& (u64GuestCr0 & X86_CR0_PG)
9257	&& !(u64GuestCr0 & X86_CR0_PE))
9258	{
9259	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9260	}
9261
9262	/*
9263	* CR4.
9264	*/
9265	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9266	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9267	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9268
9269	uint64_t u64GuestCr4;
9270	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR4, &u64GuestCr4);
9271	AssertRC(rc);
9272	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9273	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9274
9275	/*
9276	* IA32_DEBUGCTL MSR.
9277	*/
9278	uint64_t u64Val;
9279	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9280	AssertRC(rc);
9281	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9282	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9283	{
9284	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9285	}
9286	uint64_t u64DebugCtlMsr = u64Val;
9287
9288	#ifdef VBOX_STRICT
9289	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9290	AssertRC(rc);
9291	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9292	#endif
9293	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9294
9295	/*
9296	* RIP and RFLAGS.
9297	*/
9298	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RIP, &u64Val);
9299	AssertRC(rc);
9300	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9301	if ( !fLongModeGuest
9302	\|\| !pCtx->cs.Attr.n.u1Long)
9303	{
9304	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9305	}
9306	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9307	* must be identical if the "IA-32e mode guest" VM-entry
9308	* control is 1 and CS.L is 1. No check applies if the
9309	* CPU supports 64 linear-address bits. */
9310
9311	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9312	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9313	AssertRC(rc);
9314	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9315	VMX_IGS_RFLAGS_RESERVED);
9316	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9317	uint32_t const u32Eflags = u64Val;
9318
9319	if ( fLongModeGuest
9320	\|\| ( fUnrestrictedGuest
9321	&& !(u64GuestCr0 & X86_CR0_PE)))
9322	{
9323	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9324	}
9325
9326	uint32_t u32EntryInfo;
9327	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9328	AssertRC(rc);
9329	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9330	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9331
9332	/*
9333	* 64-bit checks.
9334	*/
9335	if (fLongModeGuest)
9336	{
9337	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9338	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9339	}
9340
9341	if ( !fLongModeGuest
9342	&& (u64GuestCr4 & X86_CR4_PCIDE))
9343	{
9344	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9345	}
9346
9347	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9348	* 51:32 beyond the processor's physical-address width are 0. */
9349
9350	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9351	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9352	{
9353	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9354	}
9355
9356	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9357	AssertRC(rc);
9358	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9359
9360	rc = VMXReadVmcsNw(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9361	AssertRC(rc);
9362	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9363
9364	/*
9365	* PERF_GLOBAL MSR.
9366	*/
9367	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9368	{
9369	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9370	AssertRC(rc);
9371	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9372	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9373	}
9374
9375	/*
9376	* PAT MSR.
9377	*/
9378	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9379	{
9380	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9381	AssertRC(rc);
9382	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9383	for (unsigned i = 0; i < 8; i++)
9384	{
9385	uint8_t u8Val = (u64Val & 0xff);
9386	if ( u8Val != 0 /* UC */
9387	&& u8Val != 1 /* WC */
9388	&& u8Val != 4 /* WT */
9389	&& u8Val != 5 /* WP */
9390	&& u8Val != 6 /* WB */
9391	&& u8Val != 7 /* UC- */)
9392	{
9393	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9394	}
9395	u64Val >>= 8;
9396	}
9397	}
9398
9399	/*
9400	* EFER MSR.
9401	*/
9402	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9403	{
9404	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9405	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9406	AssertRC(rc);
9407	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9408	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9409	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9410	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9411	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9412	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9413	* iemVmxVmentryCheckGuestState(). */
9414	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9415	\|\| !(u64GuestCr0 & X86_CR0_PG)
9416	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9417	VMX_IGS_EFER_LMA_LME_MISMATCH);
9418	}
9419
9420	/*
9421	* Segment registers.
9422	*/
9423	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9424	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9425	if (!(u32Eflags & X86_EFL_VM))
9426	{
9427	/* CS */
9428	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9429	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9430	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9431	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9432	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9433	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9434	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9435	/* CS cannot be loaded with NULL in protected mode. */
9436	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9437	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9438	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9439	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9440	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9441	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9442	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9443	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9444	else
9445	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9446
9447	/* SS */
9448	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9449	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9450	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9451	if ( !(pCtx->cr0 & X86_CR0_PE)
9452	\|\| pCtx->cs.Attr.n.u4Type == 3)
9453	{
9454	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9455	}
9456	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9457	{
9458	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9459	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9460	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9461	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9462	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9463	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9464	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9465	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9466	}
9467
9468	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9469	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9470	{
9471	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9472	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9473	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9474	\|\| pCtx->ds.Attr.n.u4Type > 11
9475	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9476	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9477	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9478	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9479	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9480	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9481	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9482	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9483	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9484	}
9485	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9486	{
9487	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9488	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9489	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9490	\|\| pCtx->es.Attr.n.u4Type > 11
9491	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9492	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9493	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9494	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9495	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9496	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9497	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9498	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9499	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9500	}
9501	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9502	{
9503	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9504	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9505	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9506	\|\| pCtx->fs.Attr.n.u4Type > 11
9507	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9508	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9509	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9510	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9511	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9512	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9513	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9514	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9515	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9516	}
9517	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9518	{
9519	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9520	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9521	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9522	\|\| pCtx->gs.Attr.n.u4Type > 11
9523	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9524	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9525	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9526	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9527	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9528	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9529	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9530	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9531	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9532	}
9533	/* 64-bit capable CPUs. */
9534	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9535	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9536	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9537	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9538	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9539	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9540	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9541	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9542	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9543	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9544	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9545	}
9546	else
9547	{
9548	/* V86 mode checks. */
9549	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9550	if (pVmcsInfo->RealMode.fRealOnV86Active)
9551	{
9552	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9553	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9554	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9555	}
9556	else
9557	{
9558	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9559	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9560	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9561	}
9562
9563	/* CS */
9564	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9565	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9566	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9567	/* SS */
9568	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9569	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9570	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9571	/* DS */
9572	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9573	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9574	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9575	/* ES */
9576	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9577	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9578	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9579	/* FS */
9580	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9581	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9582	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9583	/* GS */
9584	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9585	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9586	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9587	/* 64-bit capable CPUs. */
9588	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9589	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9590	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9591	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9592	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9593	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9594	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9595	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9596	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9597	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9598	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9599	}
9600
9601	/*
9602	* TR.
9603	*/
9604	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9605	/* 64-bit capable CPUs. */
9606	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9607	if (fLongModeGuest)
9608	{
9609	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9610	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9611	}
9612	else
9613	{
9614	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9615	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9616	VMX_IGS_TR_ATTR_TYPE_INVALID);
9617	}
9618	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9619	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9620	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9621	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9622	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9623	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9624	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9625	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9626
9627	/*
9628	* GDTR and IDTR (64-bit capable checks).
9629	*/
9630	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9631	AssertRC(rc);
9632	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9633
9634	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9635	AssertRC(rc);
9636	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9637
9638	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9639	AssertRC(rc);
9640	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9641
9642	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9643	AssertRC(rc);
9644	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9645
9646	/*
9647	* Guest Non-Register State.
9648	*/
9649	/* Activity State. */
9650	uint32_t u32ActivityState;
9651	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9652	AssertRC(rc);
9653	HMVMX_CHECK_BREAK( !u32ActivityState
9654	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9655	VMX_IGS_ACTIVITY_STATE_INVALID);
9656	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9657	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9658	uint32_t u32IntrState;
9659	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9660	AssertRC(rc);
9661	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9662	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9663	{
9664	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9665	}
9666
9667	/** @todo Activity state and injecting interrupts. Left as a todo since we
9668	* currently don't use activity states but ACTIVE. */
9669
9670	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9671	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9672
9673	/* Guest interruptibility-state. */
9674	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9675	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9676	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9677	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9678	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9679	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9680	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9681	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9682	{
9683	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9684	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9685	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9686	}
9687	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9688	{
9689	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9690	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9691	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9692	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9693	}
9694	/** @todo Assumes the processor is not in SMM. */
9695	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9696	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9697	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9698	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9699	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9700	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9701	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9702	{
9703	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9704	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9705	}
9706
9707	/* Pending debug exceptions. */
9708	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9709	AssertRC(rc);
9710	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9711	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9712	u32Val = u64Val; /* For pending debug exceptions checks below. */
9713
9714	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9715	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9716	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9717	{
9718	if ( (u32Eflags & X86_EFL_TF)
9719	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9720	{
9721	/* Bit 14 is PendingDebug.BS. */
9722	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9723	}
9724	if ( !(u32Eflags & X86_EFL_TF)
9725	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9726	{
9727	/* Bit 14 is PendingDebug.BS. */
9728	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9729	}
9730	}
9731
9732	/* VMCS link pointer. */
9733	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9734	AssertRC(rc);
9735	if (u64Val != UINT64_C(0xffffffffffffffff))
9736	{
9737	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9738	/** @todo Bits beyond the processor's physical-address width MBZ. */
9739	/** @todo SMM checks. */
9740	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
9741	Assert(pVmcsInfo->pvShadowVmcs);
9742	VMXVMCSREVID VmcsRevId;
9743	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
9744	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
9745	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
9746	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
9747	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
9748	}
9749
9750	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9751	* not using nested paging? */
9752	if ( pVM->hm.s.fNestedPaging
9753	&& !fLongModeGuest
9754	&& CPUMIsGuestInPAEModeEx(pCtx))
9755	{
9756	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9757	AssertRC(rc);
9758	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9759
9760	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9761	AssertRC(rc);
9762	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9763
9764	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9765	AssertRC(rc);
9766	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9767
9768	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9769	AssertRC(rc);
9770	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9771	}
9772
9773	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9774	if (uError == VMX_IGS_ERROR)
9775	uError = VMX_IGS_REASON_NOT_FOUND;
9776	} while (0);
9777
9778	pVCpu->hm.s.u32HMError = uError;
9779	return uError;
9780
9781	#undef HMVMX_ERROR_BREAK
9782	#undef HMVMX_CHECK_BREAK
9783	}
9784
9785
9786	/**
9787	* Map the APIC-access page for virtualizing APIC accesses.
9788	*
9789	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
9790	* this not done as part of exporting guest state, see @bugref{8721}.
9791	*
9792	* @returns VBox status code.
9793	* @param pVCpu The cross context virtual CPU structure.
9794	*/
9795	static int hmR0VmxMapHCApicAccessPage(PVMCPUCC pVCpu)
9796	{
9797	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9798	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9799
9800	Assert(PDMHasApic(pVM));
9801	Assert(u64MsrApicBase);
9802
9803	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9804	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9805
9806	/* Unalias the existing mapping. */
9807	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9808	AssertRCReturn(rc, rc);
9809
9810	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9811	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9812	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9813	AssertRCReturn(rc, rc);
9814
9815	/* Update the per-VCPU cache of the APIC base MSR. */
9816	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9817	return VINF_SUCCESS;
9818	}
9819
9820
9821	/**
9822	* Worker function passed to RTMpOnSpecific() that is to be called on the target
9823	* CPU.
9824	*
9825	* @param idCpu The ID for the CPU the function is called on.
9826	* @param pvUser1 Null, not used.
9827	* @param pvUser2 Null, not used.
9828	*/
9829	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
9830	{
9831	RT_NOREF3(idCpu, pvUser1, pvUser2);
9832	VMXDispatchHostNmi();
9833	}
9834
9835
9836	/**
9837	* Dispatching an NMI on the host CPU that received it.
9838	*
9839	* @returns VBox status code.
9840	* @param pVCpu The cross context virtual CPU structure.
9841	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
9842	* executing when receiving the host NMI in VMX non-root
9843	* operation.
9844	*/
9845	static int hmR0VmxExitHostNmi(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
9846	{
9847	RTCPUID const idCpu = pVmcsInfo->idHostCpuExec;
9848	Assert(idCpu != NIL_RTCPUID);
9849
9850	/*
9851	* We don't want to delay dispatching the NMI any more than we have to. However,
9852	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
9853	* after executing guest or nested-guest code for the following reasons:
9854	*
9855	* - We would need to perform VMREADs with interrupts disabled and is orders of
9856	* magnitude worse when we run as a nested hypervisor without VMCS shadowing
9857	* supported by the host hypervisor.
9858	*
9859	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
9860	* longer period of time just for handling an edge case like host NMIs which do
9861	* not occur nearly as frequently as other VM-exits.
9862	*
9863	* Let's cover the most likely scenario first. Check if we are on the target CPU
9864	* and dispatch the NMI right away. This should be much faster than calling into
9865	* RTMpOnSpecific() machinery.
9866	*/
9867	bool fDispatched = false;
9868	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
9869	if (idCpu == RTMpCpuId())
9870	{
9871	VMXDispatchHostNmi();
9872	fDispatched = true;
9873	}
9874	ASMSetFlags(fEFlags);
9875	if (fDispatched)
9876	{
9877	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9878	return VINF_SUCCESS;
9879	}
9880
9881	/*
9882	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
9883	* there should be no race or recursion even if we are unlucky enough to be preempted
9884	* (to the target CPU) without dispatching the host NMI above.
9885	*/
9886	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
9887	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
9888	}
9889
9890
9891	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9892	/**
9893	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9894	* nested-guest using hardware-assisted VMX.
9895	*
9896	* @param pVCpu The cross context virtual CPU structure.
9897	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9898	* @param pVmcsInfoGst The guest VMCS info. object.
9899	*/
9900	static void hmR0VmxMergeMsrBitmapNested(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9901	{
9902	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
9903	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9904	Assert(pu64MsrBitmap);
9905
9906	/*
9907	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9908	* MSR that is intercepted by the guest is also intercepted while executing the
9909	* nested-guest using hardware-assisted VMX.
9910	*
9911	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
9912	* nested-guest VM-exit even if the outer guest is not intercepting some
9913	* MSRs. We cannot assume the caller has initialized the nested-guest
9914	* MSR bitmap in this case.
9915	*
9916	* The nested hypervisor may also switch whether it uses MSR bitmaps for
9917	* each VM-entry, hence initializing it once per-VM while setting up the
9918	* nested-guest VMCS is not sufficient.
9919	*/
9920	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9921	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
9922	{
9923	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9924	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9925	Assert(pu64MsrBitmapNstGst);
9926	Assert(pu64MsrBitmapGst);
9927
9928	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
9929	for (uint32_t i = 0; i < cFrags; i++)
9930	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9931	}
9932	else
9933	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
9934	}
9935
9936
9937	/**
9938	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9939	* hardware-assisted VMX execution of the nested-guest.
9940	*
9941	* For a guest, we don't modify these controls once we set up the VMCS and hence
9942	* this function is never called.
9943	*
9944	* For nested-guests since the nested hypervisor provides these controls on every
9945	* nested-guest VM-entry and could potentially change them everytime we need to
9946	* merge them before every nested-guest VM-entry.
9947	*
9948	* @returns VBox status code.
9949	* @param pVCpu The cross context virtual CPU structure.
9950	*/
9951	static int hmR0VmxMergeVmcsNested(PVMCPUCC pVCpu)
9952	{
9953	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9954	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9955	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9956	Assert(pVmcsNstGst);
9957
9958	/*
9959	* Merge the controls with the requirements of the guest VMCS.
9960	*
9961	* We do not need to validate the nested-guest VMX features specified in the nested-guest
9962	* VMCS with the features supported by the physical CPU as it's already done by the
9963	* VMLAUNCH/VMRESUME instruction emulation.
9964	*
9965	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
9966	* derived from the VMX features supported by the physical CPU.
9967	*/
9968
9969	/* Pin-based VM-execution controls. */
9970	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9971
9972	/* Processor-based VM-execution controls. */
9973	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9974	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9975	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9976	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9977	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9978
9979	/* Secondary processor-based VM-execution controls. */
9980	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9981	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9982	\| VMX_PROC_CTLS2_INVPCID
9983	\| VMX_PROC_CTLS2_VMCS_SHADOWING
9984	\| VMX_PROC_CTLS2_RDTSCP
9985	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9986	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9987	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9988	\| VMX_PROC_CTLS2_VMFUNC));
9989
9990	/*
9991	* VM-entry controls:
9992	* These controls contains state that depends on the nested-guest state (primarily
9993	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
9994	* VM-exit. Although the nested hypervisor cannot change it, we need to in order to
9995	* properly continue executing the nested-guest if the EFER MSR changes but does not
9996	* cause a nested-guest VM-exits.
9997	*
9998	* VM-exit controls:
9999	* These controls specify the host state on return. We cannot use the controls from
10000	* the nested hypervisor state as is as it would contain the guest state rather than
10001	* the host state. Since the host state is subject to change (e.g. preemption, trips
10002	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10003	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10004	*
10005	* VM-entry MSR-load:
10006	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10007	* context by the VMLAUNCH/VMRESUME instruction emulation.
10008	*
10009	* VM-exit MSR-store:
10010	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10011	* back into the VM-exit MSR-store area.
10012	*
10013	* VM-exit MSR-load areas:
10014	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10015	* can entirely ignore what the nested hypervisor wants to load here.
10016	*/
10017
10018	/*
10019	* Exception bitmap.
10020	*
10021	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10022	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10023	* code more flexible if intercepting exceptions become more dynamic in the future we do
10024	* it as part of exporting the nested-guest state.
10025	*/
10026	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10027
10028	/*
10029	* CR0/CR4 guest/host mask.
10030	*
10031	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10032	* cause VM-exits, so we need to merge them here.
10033	*/
10034	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10035	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10036
10037	/*
10038	* Page-fault error-code mask and match.
10039	*
10040	* Although we require unrestricted guest execution (and thereby nested-paging) for
10041	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10042	* normally intercept #PFs, it might intercept them for debugging purposes.
10043	*
10044	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10045	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10046	*/
10047	uint32_t u32XcptPFMask;
10048	uint32_t u32XcptPFMatch;
10049	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10050	{
10051	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10052	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10053	}
10054	else
10055	{
10056	u32XcptPFMask = 0;
10057	u32XcptPFMatch = 0;
10058	}
10059
10060	/*
10061	* Pause-Loop exiting.
10062	*/
10063	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10064	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10065
10066	/*
10067	* Pending debug exceptions.
10068	* Currently just copy whatever the nested-guest provides us.
10069	*/
10070	uint64_t const uPendingDbgXcpts = pVmcsNstGst->u64GuestPendingDbgXcpts.u;
10071
10072	/*
10073	* I/O Bitmap.
10074	*
10075	* We do not use the I/O bitmap that may be provided by the nested hypervisor as we always
10076	* intercept all I/O port accesses.
10077	*/
10078	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10079	Assert(!(u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS));
10080
10081	/*
10082	* VMCS shadowing.
10083	*
10084	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10085	* enabled while executing the nested-guest.
10086	*/
10087	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10088
10089	/*
10090	* APIC-access page.
10091	*/
10092	RTHCPHYS HCPhysApicAccess;
10093	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10094	{
10095	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10096	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10097
10098	/** @todo NSTVMX: This is not really correct but currently is required to make
10099	* things work. We need to re-enable the page handler when we fallback to
10100	* IEM execution of the nested-guest! */
10101	PGMHandlerPhysicalPageTempOff(pVM, GCPhysApicAccess, GCPhysApicAccess);
10102
10103	void *pvPage;
10104	PGMPAGEMAPLOCK PgLockApicAccess;
10105	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgLockApicAccess);
10106	if (RT_SUCCESS(rc))
10107	{
10108	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10109	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10110
10111	/** @todo Handle proper releasing of page-mapping lock later. */
10112	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockApicAccess);
10113	}
10114	else
10115	return rc;
10116	}
10117	else
10118	HCPhysApicAccess = 0;
10119
10120	/*
10121	* Virtual-APIC page and TPR threshold.
10122	*/
10123	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10124	RTHCPHYS HCPhysVirtApic;
10125	uint32_t u32TprThreshold;
10126	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10127	{
10128	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10129	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10130
10131	void *pvPage;
10132	PGMPAGEMAPLOCK PgLockVirtApic;
10133	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &PgLockVirtApic);
10134	if (RT_SUCCESS(rc))
10135	{
10136	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10137	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10138
10139	/** @todo Handle proper releasing of page-mapping lock later. */
10140	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgLockVirtApic);
10141	}
10142	else
10143	return rc;
10144
10145	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10146	}
10147	else
10148	{
10149	HCPhysVirtApic = 0;
10150	u32TprThreshold = 0;
10151
10152	/*
10153	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10154	* used by the nested hypervisor. Preventing MMIO accesses to the physical APIC will
10155	* be taken care of by EPT/shadow paging.
10156	*/
10157	if (pVM->hm.s.fAllow64BitGuests)
10158	{
10159	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10160	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10161	}
10162	}
10163
10164	/*
10165	* Validate basic assumptions.
10166	*/
10167	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10168	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10169	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10170
10171	/*
10172	* Commit it to the nested-guest VMCS.
10173	*/
10174	int rc = VINF_SUCCESS;
10175	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10176	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10177	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10178	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10179	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10180	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10181	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10182	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10183	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10184	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10185	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10186	rc \|= VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10187	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10188	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10189	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10190	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10191	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10192	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10193	{
10194	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10195	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10196	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10197	}
10198	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10199	{
10200	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10201	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10202	}
10203	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10204	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10205	rc \|= VMXWriteVmcsNw(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, uPendingDbgXcpts);
10206	AssertRC(rc);
10207
10208	/*
10209	* Update the nested-guest VMCS cache.
10210	*/
10211	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10212	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10213	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10214	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10215	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10216	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10217	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10218	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10219	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10220
10221	/*
10222	* We need to flush the TLB if we are switching the APIC-access page address.
10223	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10224	*/
10225	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10226	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10227
10228	/*
10229	* MSR bitmap.
10230	*
10231	* The MSR bitmap address has already been initialized while setting up the nested-guest
10232	* VMCS, here we need to merge the MSR bitmaps.
10233	*/
10234	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10235	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10236
10237	return VINF_SUCCESS;
10238	}
10239	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10240
10241
10242	/**
10243	* Does the preparations before executing guest code in VT-x.
10244	*
10245	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10246	* recompiler/IEM. We must be cautious what we do here regarding committing
10247	* guest-state information into the VMCS assuming we assuredly execute the
10248	* guest in VT-x mode.
10249	*
10250	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10251	* the common-state (TRPM/forceflags), we must undo those changes so that the
10252	* recompiler/IEM can (and should) use them when it resumes guest execution.
10253	* Otherwise such operations must be done when we can no longer exit to ring-3.
10254	*
10255	* @returns Strict VBox status code (i.e. informational status codes too).
10256	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10257	* have been disabled.
10258	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10259	* pending events).
10260	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10261	* double-fault into the guest.
10262	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10263	* dispatched directly.
10264	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10265	*
10266	* @param pVCpu The cross context virtual CPU structure.
10267	* @param pVmxTransient The VMX-transient structure.
10268	* @param fStepping Whether we are single-stepping the guest in the
10269	* hypervisor debugger. Makes us ignore some of the reasons
10270	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10271	* if event dispatching took place.
10272	*/
10273	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10274	{
10275	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10276
10277	Log4Func(("fIsNested=%RTbool fStepping=%RTbool\n", pVmxTransient->fIsNestedGuest, fStepping));
10278
10279	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10280	if (pVmxTransient->fIsNestedGuest)
10281	{
10282	RT_NOREF2(pVCpu, fStepping);
10283	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10284	return VINF_EM_RESCHEDULE_REM;
10285	}
10286	#endif
10287
10288	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10289	PGMRZDynMapFlushAutoSet(pVCpu);
10290	#endif
10291
10292	/*
10293	* Check and process force flag actions, some of which might require us to go back to ring-3.
10294	*/
10295	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10296	if (rcStrict == VINF_SUCCESS)
10297	{
10298	/* FFs don't get set all the time. */
10299	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10300	if ( pVmxTransient->fIsNestedGuest
10301	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10302	{
10303	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10304	return VINF_VMX_VMEXIT;
10305	}
10306	#endif
10307	}
10308	else
10309	return rcStrict;
10310
10311	/*
10312	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10313	*/
10314	/** @todo Doing this from ring-3 after VM setup phase causes a
10315	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
10316	* idea why atm. */
10317	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10318	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10319	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10320	&& PDMHasApic(pVM))
10321	{
10322	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10323	AssertRCReturn(rc, rc);
10324	}
10325
10326	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10327	/*
10328	* Merge guest VMCS controls with the nested-guest VMCS controls.
10329	*
10330	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10331	* saved state), we should be okay with merging controls as we initialize the
10332	* guest VMCS controls as part of VM setup phase.
10333	*/
10334	if ( pVmxTransient->fIsNestedGuest
10335	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10336	{
10337	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10338	AssertRCReturn(rc, rc);
10339	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10340	}
10341	#endif
10342
10343	/*
10344	* Evaluate events to be injected into the guest.
10345	*
10346	* Events in TRPM can be injected without inspecting the guest state.
10347	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10348	* guest to cause a VM-exit the next time they are ready to receive the event.
10349	*
10350	* With nested-guests, evaluating pending events may cause VM-exits. Also, verify
10351	* that the event in TRPM that we will inject using hardware-assisted VMX is -not-
10352	* subject to interecption. Otherwise, we should have checked and injected them
10353	* manually elsewhere (IEM).
10354	*/
10355	if (TRPMHasTrap(pVCpu))
10356	{
10357	Assert(!pVmxTransient->fIsNestedGuest \|\| !CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
10358	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10359	}
10360
10361	uint32_t fIntrState;
10362	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10363
10364	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10365	/*
10366	* While evaluating pending events if something failed (unlikely) or if we were
10367	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10368	*/
10369	if (rcStrict != VINF_SUCCESS)
10370	return rcStrict;
10371	if ( pVmxTransient->fIsNestedGuest
10372	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10373	{
10374	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10375	return VINF_VMX_VMEXIT;
10376	}
10377	#else
10378	Assert(rcStrict == VINF_SUCCESS);
10379	#endif
10380
10381	/*
10382	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10383	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10384	* also result in triple-faulting the VM.
10385	*
10386	* With nested-guests, the above does not apply since unrestricted guest execution is a
10387	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10388	*/
10389	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10390	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10391	{ /* likely */ }
10392	else
10393	{
10394	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10395	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10396	return rcStrict;
10397	}
10398
10399	/*
10400	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10401	* import CR3 themselves. We will need to update them here, as even as late as the above
10402	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10403	* the below force flags to be set.
10404	*/
10405	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10406	{
10407	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10408	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10409	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10410	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10411	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10412	}
10413	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10414	{
10415	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10416	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10417	}
10418
10419	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10420	/* Paranoia. */
10421	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10422	#endif
10423
10424	/*
10425	* No longjmps to ring-3 from this point on!!!
10426	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10427	* This also disables flushing of the R0-logger instance (if any).
10428	*/
10429	VMMRZCallRing3Disable(pVCpu);
10430
10431	/*
10432	* Export the guest state bits.
10433	*
10434	* We cannot perform longjmps while loading the guest state because we do not preserve the
10435	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10436	* CPU migration.
10437	*
10438	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10439	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10440	*/
10441	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10442	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10443	{ /* likely */ }
10444	else
10445	{
10446	VMMRZCallRing3Enable(pVCpu);
10447	return rcStrict;
10448	}
10449
10450	/*
10451	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10452	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10453	* preemption disabled for a while. Since this is purely to aid the
10454	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10455	* disable interrupt on NT.
10456	*
10457	* We need to check for force-flags that could've possible been altered since we last
10458	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10459	* see @bugref{6398}).
10460	*
10461	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10462	* to ring-3 before executing guest code.
10463	*/
10464	pVmxTransient->fEFlags = ASMIntDisableFlags();
10465
10466	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10467	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10468	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10469	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10470	{
10471	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10472	{
10473	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10474	/*
10475	* If we are executing a nested-guest make sure that we should intercept subsequent
10476	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10477	* the VM-exit instruction emulation happy.
10478	*/
10479	if (pVmxTransient->fIsNestedGuest)
10480	CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
10481	#endif
10482
10483	/*
10484	* We've injected any pending events. This is really the point of no return (to ring-3).
10485	*
10486	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10487	* returns from this function, so do -not- enable them here.
10488	*/
10489	pVCpu->hm.s.Event.fPending = false;
10490	return VINF_SUCCESS;
10491	}
10492
10493	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10494	rcStrict = VINF_EM_RAW_INTERRUPT;
10495	}
10496	else
10497	{
10498	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10499	rcStrict = VINF_EM_RAW_TO_R3;
10500	}
10501
10502	ASMSetFlags(pVmxTransient->fEFlags);
10503	VMMRZCallRing3Enable(pVCpu);
10504
10505	return rcStrict;
10506	}
10507
10508
10509	/**
10510	* Final preparations before executing guest code using hardware-assisted VMX.
10511	*
10512	* We can no longer get preempted to a different host CPU and there are no returns
10513	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10514	* failures), this function is not intended to fail sans unrecoverable hardware
10515	* errors.
10516	*
10517	* @param pVCpu The cross context virtual CPU structure.
10518	* @param pVmxTransient The VMX-transient structure.
10519	*
10520	* @remarks Called with preemption disabled.
10521	* @remarks No-long-jump zone!!!
10522	*/
10523	static void hmR0VmxPreRunGuestCommitted(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10524	{
10525	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10526	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10527	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10528	Assert(!pVCpu->hm.s.Event.fPending);
10529
10530	/*
10531	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10532	*/
10533	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10534	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10535
10536	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10537	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10538	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10539	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10540
10541	if (!CPUMIsGuestFPUStateActive(pVCpu))
10542	{
10543	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10544	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10545	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10546	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10547	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10548	}
10549
10550	/*
10551	* Re-export the host state bits as we may've been preempted (only happens when
10552	* thread-context hooks are used or when the VM start function changes) or if
10553	* the host CR0 is modified while loading the guest FPU state above.
10554	*
10555	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10556	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10557	* see @bugref{8432}.
10558	*
10559	* This may also happen when switching to/from a nested-guest VMCS without leaving
10560	* ring-0.
10561	*/
10562	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10563	{
10564	hmR0VmxExportHostState(pVCpu);
10565	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10566	}
10567	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10568
10569	/*
10570	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10571	*/
10572	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10573	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10574	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10575
10576	/*
10577	* Store status of the shared guest/host debug state at the time of VM-entry.
10578	*/
10579	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10580	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10581
10582	/*
10583	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10584	* more than one conditional check. The post-run side of our code shall determine
10585	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10586	*/
10587	if (pVmcsInfo->pbVirtApic)
10588	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10589
10590	/*
10591	* Update the host MSRs values in the VM-exit MSR-load area.
10592	*/
10593	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10594	{
10595	if (pVmcsInfo->cExitMsrLoad > 0)
10596	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10597	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10598	}
10599
10600	/*
10601	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10602	* VMX-preemption timer based on the next virtual sync clock deadline.
10603	*/
10604	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10605	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10606	{
10607	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10608	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10609	}
10610
10611	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10612	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10613	if (!fIsRdtscIntercepted)
10614	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10615	else
10616	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10617
10618	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10619	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10620	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10621	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Record the error reporting info. with the current host CPU. */
10622	pVmcsInfo->idHostCpuState = idCurrentCpu; /* Record the CPU for which the host-state has been exported. */
10623	pVmcsInfo->idHostCpuExec = idCurrentCpu; /* Record the CPU on which we shall execute. */
10624
10625	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10626
10627	TMNotifyStartOfExecution(pVM, pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10628	as we're about to start executing the guest. */
10629
10630	/*
10631	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10632	*
10633	* This is done this late as updating the TSC offsetting/preemption timer above
10634	* figures out if we can skip intercepting RDTSCP by calculating the number of
10635	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10636	*/
10637	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10638	&& !fIsRdtscIntercepted)
10639	{
10640	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10641
10642	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10643	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10644	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10645	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10646	AssertRC(rc);
10647	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10648	pVmxTransient->fRemoveTscAuxMsr = true;
10649	}
10650
10651	#ifdef VBOX_STRICT
10652	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10653	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10654	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10655	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10656	#endif
10657
10658	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10659	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10660	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10661	* see @bugref{9180#c54}. */
10662	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10663	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10664	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10665	#endif
10666	}
10667
10668
10669	/**
10670	* First C routine invoked after running guest code using hardware-assisted VMX.
10671	*
10672	* @param pVCpu The cross context virtual CPU structure.
10673	* @param pVmxTransient The VMX-transient structure.
10674	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10675	*
10676	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10677	*
10678	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10679	* unconditionally when it is safe to do so.
10680	*/
10681	static void hmR0VmxPostRunGuest(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10682	{
10683	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10684
10685	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10686	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10687	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10688	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10689	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10690	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10691
10692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10693	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10694	{
10695	uint64_t uGstTsc;
10696	if (!pVmxTransient->fIsNestedGuest)
10697	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10698	else
10699	{
10700	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10701	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10702	}
10703	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10704	}
10705
10706	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10707	TMNotifyEndOfExecution(pVCpu->CTX_SUFF(pVM), pVCpu); /* Notify TM that the guest is no longer running. */
10708	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10709
10710	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10711	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10712	#ifdef VBOX_STRICT
10713	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10714	#endif
10715	Assert(!ASMIntAreEnabled());
10716	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10717	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10718
10719	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
10720	/*
10721	* Clean all the VMCS fields in the transient structure before reading
10722	* anything from the VMCS.
10723	*/
10724	pVmxTransient->uExitReason = 0;
10725	pVmxTransient->uExitIntErrorCode = 0;
10726	pVmxTransient->uExitQual = 0;
10727	pVmxTransient->uGuestLinearAddr = 0;
10728	pVmxTransient->uExitIntInfo = 0;
10729	pVmxTransient->cbExitInstr = 0;
10730	pVmxTransient->ExitInstrInfo.u = 0;
10731	pVmxTransient->uEntryIntInfo = 0;
10732	pVmxTransient->uEntryXcptErrorCode = 0;
10733	pVmxTransient->cbEntryInstr = 0;
10734	pVmxTransient->uIdtVectoringInfo = 0;
10735	pVmxTransient->uIdtVectoringErrorCode = 0;
10736	#endif
10737
10738	/*
10739	* Save the basic VM-exit reason and check if the VM-entry failed.
10740	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10741	*/
10742	uint32_t uExitReason;
10743	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10744	AssertRC(rc);
10745	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10746	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10747
10748	/*
10749	* Log the VM-exit before logging anything else as otherwise it might be a
10750	* tad confusing what happens before and after the world-switch.
10751	*/
10752	HMVMX_LOG_EXIT(pVCpu, uExitReason);
10753
10754	/*
10755	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
10756	* bitmap permissions, if it was added before VM-entry.
10757	*/
10758	if (pVmxTransient->fRemoveTscAuxMsr)
10759	{
10760	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10761	pVmxTransient->fRemoveTscAuxMsr = false;
10762	}
10763
10764	/*
10765	* Check if VMLAUNCH/VMRESUME succeeded.
10766	* If this failed, we cause a guru meditation and cease further execution.
10767	*
10768	* However, if we are executing a nested-guest we might fail if we use the
10769	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10770	*/
10771	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10772	{
10773	/*
10774	* Update the VM-exit history array here even if the VM-entry failed due to:
10775	* - Invalid guest state.
10776	* - MSR loading.
10777	* - Machine-check event.
10778	*
10779	* In any of the above cases we will still have a "valid" VM-exit reason
10780	* despite @a fVMEntryFailed being false.
10781	*
10782	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10783	*
10784	* Note! We don't have CS or RIP at this point. Will probably address that later
10785	* by amending the history entry added here.
10786	*/
10787	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10788	UINT64_MAX, uHostTsc);
10789
10790	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10791	{
10792	VMMRZCallRing3Enable(pVCpu);
10793
10794	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10795	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10796
10797	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
10798	hmR0VmxReadAllRoFieldsVmcs(pVmxTransient);
10799	#endif
10800	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10801	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10802	AssertRC(rc);
10803	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10804	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10805	AssertRC(rc);
10806	#else
10807	/*
10808	* Import the guest-interruptibility state always as we need it while evaluating
10809	* injecting events on re-entry.
10810	*
10811	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10812	* checking for real-mode while exporting the state because all bits that cause
10813	* mode changes wrt CR0 are intercepted.
10814	*/
10815	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10816	AssertRC(rc);
10817	#endif
10818
10819	/*
10820	* Sync the TPR shadow with our APIC state.
10821	*/
10822	if ( !pVmxTransient->fIsNestedGuest
10823	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10824	{
10825	Assert(pVmcsInfo->pbVirtApic);
10826	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10827	{
10828	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10829	AssertRC(rc);
10830	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10831	}
10832	}
10833
10834	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10835	return;
10836	}
10837	}
10838	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10839	else if (pVmxTransient->fIsNestedGuest)
10840	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10841	#endif
10842	else
10843	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10844
10845	VMMRZCallRing3Enable(pVCpu);
10846	}
10847
10848
10849	/**
10850	* Runs the guest code using hardware-assisted VMX the normal way.
10851	*
10852	* @returns VBox status code.
10853	* @param pVCpu The cross context virtual CPU structure.
10854	* @param pcLoops Pointer to the number of executed loops.
10855	*/
10856	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
10857	{
10858	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10859	Assert(pcLoops);
10860	Assert(*pcLoops <= cMaxResumeLoops);
10861	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10862
10863	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10864	/*
10865	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
10866	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
10867	* guest VMCS while entering the VMX ring-0 session.
10868	*/
10869	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10870	{
10871	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
10872	if (RT_SUCCESS(rc))
10873	{ /* likely */ }
10874	else
10875	{
10876	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
10877	return rc;
10878	}
10879	}
10880	#endif
10881
10882	VMXTRANSIENT VmxTransient;
10883	RT_ZERO(VmxTransient);
10884	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10885
10886	/* Paranoia. */
10887	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10888
10889	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10890	for (;;)
10891	{
10892	Assert(!HMR0SuspendPending());
10893	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10894	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10895
10896	/*
10897	* Preparatory work for running nested-guest code, this may force us to
10898	* return to ring-3.
10899	*
10900	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10901	*/
10902	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10903	if (rcStrict != VINF_SUCCESS)
10904	break;
10905
10906	/* Interrupts are disabled at this point! */
10907	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10908	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10909	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10910	/* Interrupts are re-enabled at this point! */
10911
10912	/*
10913	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10914	*/
10915	if (RT_SUCCESS(rcRun))
10916	{ /* very likely */ }
10917	else
10918	{
10919	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10920	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10921	return rcRun;
10922	}
10923
10924	/*
10925	* Profile the VM-exit.
10926	*/
10927	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10928	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10929	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10930	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10931	HMVMX_START_EXIT_DISPATCH_PROF();
10932
10933	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10934
10935	/*
10936	* Handle the VM-exit.
10937	*/
10938	#ifdef HMVMX_USE_FUNCTION_TABLE
10939	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10940	#else
10941	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10942	#endif
10943	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10944	if (rcStrict == VINF_SUCCESS)
10945	{
10946	if (++(*pcLoops) <= cMaxResumeLoops)
10947	continue;
10948	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10949	rcStrict = VINF_EM_RAW_INTERRUPT;
10950	}
10951	break;
10952	}
10953
10954	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10955	return rcStrict;
10956	}
10957
10958
10959	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10960	/**
10961	* Runs the nested-guest code using hardware-assisted VMX.
10962	*
10963	* @returns VBox status code.
10964	* @param pVCpu The cross context virtual CPU structure.
10965	* @param pcLoops Pointer to the number of executed loops.
10966	*
10967	* @sa hmR0VmxRunGuestCodeNormal.
10968	*/
10969	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
10970	{
10971	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10972	Assert(pcLoops);
10973	Assert(*pcLoops <= cMaxResumeLoops);
10974	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10975
10976	/*
10977	* Switch to the nested-guest VMCS as we may have transitioned from executing the
10978	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
10979	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
10980	*/
10981	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10982	{
10983	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
10984	if (RT_SUCCESS(rc))
10985	{ /* likely */ }
10986	else
10987	{
10988	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
10989	return rc;
10990	}
10991	}
10992
10993	VMXTRANSIENT VmxTransient;
10994	RT_ZERO(VmxTransient);
10995	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10996	VmxTransient.fIsNestedGuest = true;
10997
10998	/* Paranoia. */
10999	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11000
11001	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11002	for (;;)
11003	{
11004	Assert(!HMR0SuspendPending());
11005	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11006	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11007
11008	/*
11009	* Preparatory work for running guest code, this may force us to
11010	* return to ring-3.
11011	*
11012	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11013	*/
11014	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11015	if (rcStrict != VINF_SUCCESS)
11016	break;
11017
11018	/* Interrupts are disabled at this point! */
11019	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11020	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11021	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11022	/* Interrupts are re-enabled at this point! */
11023
11024	/*
11025	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11026	*/
11027	if (RT_SUCCESS(rcRun))
11028	{ /* very likely */ }
11029	else
11030	{
11031	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11032	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11033	return rcRun;
11034	}
11035
11036	/*
11037	* Profile the VM-exit.
11038	*/
11039	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11040	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11041	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11042	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11043	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11044	HMVMX_START_EXIT_DISPATCH_PROF();
11045
11046	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11047
11048	/*
11049	* Handle the VM-exit.
11050	*/
11051	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11052	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11053	if (rcStrict == VINF_SUCCESS)
11054	{
11055	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11056	{
11057	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11058	rcStrict = VINF_VMX_VMEXIT;
11059	}
11060	else
11061	{
11062	if (++(*pcLoops) <= cMaxResumeLoops)
11063	continue;
11064	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11065	rcStrict = VINF_EM_RAW_INTERRUPT;
11066	}
11067	}
11068	else
11069	Assert(rcStrict != VINF_VMX_VMEXIT);
11070	break;
11071	}
11072
11073	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11074	return rcStrict;
11075	}
11076	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11077
11078
11079	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11080	* probes.
11081	*
11082	* The following few functions and associated structure contains the bloat
11083	* necessary for providing detailed debug events and dtrace probes as well as
11084	* reliable host side single stepping. This works on the principle of
11085	* "subclassing" the normal execution loop and workers. We replace the loop
11086	* method completely and override selected helpers to add necessary adjustments
11087	* to their core operation.
11088	*
11089	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11090	* any performance for debug and analysis features.
11091	*
11092	* @{
11093	*/
11094
11095	/**
11096	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11097	* the debug run loop.
11098	*/
11099	typedef struct VMXRUNDBGSTATE
11100	{
11101	/** The RIP we started executing at. This is for detecting that we stepped. */
11102	uint64_t uRipStart;
11103	/** The CS we started executing with. */
11104	uint16_t uCsStart;
11105
11106	/** Whether we've actually modified the 1st execution control field. */
11107	bool fModifiedProcCtls : 1;
11108	/** Whether we've actually modified the 2nd execution control field. */
11109	bool fModifiedProcCtls2 : 1;
11110	/** Whether we've actually modified the exception bitmap. */
11111	bool fModifiedXcptBitmap : 1;
11112
11113	/** We desire the modified the CR0 mask to be cleared. */
11114	bool fClearCr0Mask : 1;
11115	/** We desire the modified the CR4 mask to be cleared. */
11116	bool fClearCr4Mask : 1;
11117	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11118	uint32_t fCpe1Extra;
11119	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11120	uint32_t fCpe1Unwanted;
11121	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11122	uint32_t fCpe2Extra;
11123	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11124	uint32_t bmXcptExtra;
11125	/** The sequence number of the Dtrace provider settings the state was
11126	* configured against. */
11127	uint32_t uDtraceSettingsSeqNo;
11128	/** VM-exits to check (one bit per VM-exit). */
11129	uint32_t bmExitsToCheck[3];
11130
11131	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11132	uint32_t fProcCtlsInitial;
11133	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11134	uint32_t fProcCtls2Initial;
11135	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11136	uint32_t bmXcptInitial;
11137	} VMXRUNDBGSTATE;
11138	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11139	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11140
11141
11142	/**
11143	* Initializes the VMXRUNDBGSTATE structure.
11144	*
11145	* @param pVCpu The cross context virtual CPU structure of the
11146	* calling EMT.
11147	* @param pVmxTransient The VMX-transient structure.
11148	* @param pDbgState The debug state to initialize.
11149	*/
11150	static void hmR0VmxRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11151	{
11152	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11153	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11154
11155	pDbgState->fModifiedProcCtls = false;
11156	pDbgState->fModifiedProcCtls2 = false;
11157	pDbgState->fModifiedXcptBitmap = false;
11158	pDbgState->fClearCr0Mask = false;
11159	pDbgState->fClearCr4Mask = false;
11160	pDbgState->fCpe1Extra = 0;
11161	pDbgState->fCpe1Unwanted = 0;
11162	pDbgState->fCpe2Extra = 0;
11163	pDbgState->bmXcptExtra = 0;
11164	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11165	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11166	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11167	}
11168
11169
11170	/**
11171	* Updates the VMSC fields with changes requested by @a pDbgState.
11172	*
11173	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11174	* immediately before executing guest code, i.e. when interrupts are disabled.
11175	* We don't check status codes here as we cannot easily assert or return in the
11176	* latter case.
11177	*
11178	* @param pVCpu The cross context virtual CPU structure.
11179	* @param pVmxTransient The VMX-transient structure.
11180	* @param pDbgState The debug state.
11181	*/
11182	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11183	{
11184	/*
11185	* Ensure desired flags in VMCS control fields are set.
11186	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11187	*
11188	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11189	* there should be no stale data in pCtx at this point.
11190	*/
11191	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11192	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11193	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11194	{
11195	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11196	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11197	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11198	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11199	pDbgState->fModifiedProcCtls = true;
11200	}
11201
11202	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11203	{
11204	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11205	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11206	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11207	pDbgState->fModifiedProcCtls2 = true;
11208	}
11209
11210	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11211	{
11212	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11213	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11214	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11215	pDbgState->fModifiedXcptBitmap = true;
11216	}
11217
11218	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11219	{
11220	pVmcsInfo->u64Cr0Mask = 0;
11221	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR0_MASK, 0);
11222	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11223	}
11224
11225	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11226	{
11227	pVmcsInfo->u64Cr4Mask = 0;
11228	VMXWriteVmcsNw(VMX_VMCS_CTRL_CR4_MASK, 0);
11229	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11230	}
11231
11232	NOREF(pVCpu);
11233	}
11234
11235
11236	/**
11237	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11238	* re-entry next time around.
11239	*
11240	* @returns Strict VBox status code (i.e. informational status codes too).
11241	* @param pVCpu The cross context virtual CPU structure.
11242	* @param pVmxTransient The VMX-transient structure.
11243	* @param pDbgState The debug state.
11244	* @param rcStrict The return code from executing the guest using single
11245	* stepping.
11246	*/
11247	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11248	VBOXSTRICTRC rcStrict)
11249	{
11250	/*
11251	* Restore VM-exit control settings as we may not reenter this function the
11252	* next time around.
11253	*/
11254	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11255
11256	/* We reload the initial value, trigger what we can of recalculations the
11257	next time around. From the looks of things, that's all that's required atm. */
11258	if (pDbgState->fModifiedProcCtls)
11259	{
11260	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11261	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11262	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11263	AssertRC(rc2);
11264	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11265	}
11266
11267	/* We're currently the only ones messing with this one, so just restore the
11268	cached value and reload the field. */
11269	if ( pDbgState->fModifiedProcCtls2
11270	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11271	{
11272	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11273	AssertRC(rc2);
11274	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11275	}
11276
11277	/* If we've modified the exception bitmap, we restore it and trigger
11278	reloading and partial recalculation the next time around. */
11279	if (pDbgState->fModifiedXcptBitmap)
11280	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11281
11282	return rcStrict;
11283	}
11284
11285
11286	/**
11287	* Configures VM-exit controls for current DBGF and DTrace settings.
11288	*
11289	* This updates @a pDbgState and the VMCS execution control fields to reflect
11290	* the necessary VM-exits demanded by DBGF and DTrace.
11291	*
11292	* @param pVCpu The cross context virtual CPU structure.
11293	* @param pVmxTransient The VMX-transient structure. May update
11294	* fUpdatedTscOffsettingAndPreemptTimer.
11295	* @param pDbgState The debug state.
11296	*/
11297	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11298	{
11299	/*
11300	* Take down the dtrace serial number so we can spot changes.
11301	*/
11302	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11303	ASMCompilerBarrier();
11304
11305	/*
11306	* We'll rebuild most of the middle block of data members (holding the
11307	* current settings) as we go along here, so start by clearing it all.
11308	*/
11309	pDbgState->bmXcptExtra = 0;
11310	pDbgState->fCpe1Extra = 0;
11311	pDbgState->fCpe1Unwanted = 0;
11312	pDbgState->fCpe2Extra = 0;
11313	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11314	pDbgState->bmExitsToCheck[i] = 0;
11315
11316	/*
11317	* Software interrupts (INT XXh) - no idea how to trigger these...
11318	*/
11319	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11320	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11321	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11322	{
11323	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11324	}
11325
11326	/*
11327	* INT3 breakpoints - triggered by #BP exceptions.
11328	*/
11329	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11330	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11331
11332	/*
11333	* Exception bitmap and XCPT events+probes.
11334	*/
11335	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11336	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11337	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11338
11339	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11340	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11341	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11342	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11343	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11344	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11345	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11346	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11347	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11348	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11349	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11350	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11351	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11352	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11353	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11354	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11355	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11356	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11357
11358	if (pDbgState->bmXcptExtra)
11359	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11360
11361	/*
11362	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11363	*
11364	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11365	* So, when adding/changing/removing please don't forget to update it.
11366	*
11367	* Some of the macros are picking up local variables to save horizontal space,
11368	* (being able to see it in a table is the lesser evil here).
11369	*/
11370	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11371	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11372	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11373	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11374	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11375	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11376	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11377	} else do { } while (0)
11378	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11379	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11380	{ \
11381	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11382	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11383	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11384	} else do { } while (0)
11385	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11386	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11387	{ \
11388	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11389	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11390	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11391	} else do { } while (0)
11392	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11393	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11394	{ \
11395	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11396	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11397	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11398	} else do { } while (0)
11399
11400	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11401	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11402	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11403	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11404	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11405
11406	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11407	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11408	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11409	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11410	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11411	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11412	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11413	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11414	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11415	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11416	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11417	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11418	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11419	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11420	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11421	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11422	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11423	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11424	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11425	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11426	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11427	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11428	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11429	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11430	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11431	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11432	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11433	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11434	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11435	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11436	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11437	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11438	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11439	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11440	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11441	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11442
11443	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11444	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11445	{
11446	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11447	\| CPUMCTX_EXTRN_APIC_TPR);
11448	AssertRC(rc);
11449
11450	#if 0 /** @todo fix me */
11451	pDbgState->fClearCr0Mask = true;
11452	pDbgState->fClearCr4Mask = true;
11453	#endif
11454	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11455	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11456	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11457	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11458	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11459	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11460	require clearing here and in the loop if we start using it. */
11461	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11462	}
11463	else
11464	{
11465	if (pDbgState->fClearCr0Mask)
11466	{
11467	pDbgState->fClearCr0Mask = false;
11468	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11469	}
11470	if (pDbgState->fClearCr4Mask)
11471	{
11472	pDbgState->fClearCr4Mask = false;
11473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11474	}
11475	}
11476	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11477	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11478
11479	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11480	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11481	{
11482	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11483	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11484	}
11485	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11486	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11487
11488	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11489	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11490	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11491	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11492	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11493	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11494	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11495	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11496	#if 0 /** @todo too slow, fix handler. */
11497	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11498	#endif
11499	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11500
11501	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11502	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11503	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11504	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11505	{
11506	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11507	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11508	}
11509	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11510	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11511	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11512	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11513
11514	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11515	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11516	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11517	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11518	{
11519	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11520	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11521	}
11522	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11523	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11524	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11525	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11526
11527	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11528	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11529	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11530	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11531	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11532	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11533	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11534	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11535	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11536	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11537	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11538	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11539	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11540	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11541	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11542	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11543	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11544	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11545	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11546	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11547	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11548	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11549
11550	#undef IS_EITHER_ENABLED
11551	#undef SET_ONLY_XBM_IF_EITHER_EN
11552	#undef SET_CPE1_XBM_IF_EITHER_EN
11553	#undef SET_CPEU_XBM_IF_EITHER_EN
11554	#undef SET_CPE2_XBM_IF_EITHER_EN
11555
11556	/*
11557	* Sanitize the control stuff.
11558	*/
11559	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11560	if (pDbgState->fCpe2Extra)
11561	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11562	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11563	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11564	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11565	{
11566	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11567	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11568	}
11569
11570	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11571	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11572	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11573	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11574	}
11575
11576
11577	/**
11578	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11579	* appropriate.
11580	*
11581	* The caller has checked the VM-exit against the
11582	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11583	* already, so we don't have to do that either.
11584	*
11585	* @returns Strict VBox status code (i.e. informational status codes too).
11586	* @param pVCpu The cross context virtual CPU structure.
11587	* @param pVmxTransient The VMX-transient structure.
11588	* @param uExitReason The VM-exit reason.
11589	*
11590	* @remarks The name of this function is displayed by dtrace, so keep it short
11591	* and to the point. No longer than 33 chars long, please.
11592	*/
11593	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11594	{
11595	/*
11596	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11597	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11598	*
11599	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11600	* does. Must add/change/remove both places. Same ordering, please.
11601	*
11602	* Added/removed events must also be reflected in the next section
11603	* where we dispatch dtrace events.
11604	*/
11605	bool fDtrace1 = false;
11606	bool fDtrace2 = false;
11607	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11608	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11609	uint32_t uEventArg = 0;
11610	#define SET_EXIT(a_EventSubName) \
11611	do { \
11612	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11613	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11614	} while (0)
11615	#define SET_BOTH(a_EventSubName) \
11616	do { \
11617	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11618	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11619	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11620	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11621	} while (0)
11622	switch (uExitReason)
11623	{
11624	case VMX_EXIT_MTF:
11625	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11626
11627	case VMX_EXIT_XCPT_OR_NMI:
11628	{
11629	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11630	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11631	{
11632	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11633	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11634	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11635	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11636	{
11637	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11638	{
11639	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11640	uEventArg = pVmxTransient->uExitIntErrorCode;
11641	}
11642	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11643	switch (enmEvent1)
11644	{
11645	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11646	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11647	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11648	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11649	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11650	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11651	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11652	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11653	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11654	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11655	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11656	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11657	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11658	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11659	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11660	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11661	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11662	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11663	default: break;
11664	}
11665	}
11666	else
11667	AssertFailed();
11668	break;
11669
11670	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11671	uEventArg = idxVector;
11672	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11673	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11674	break;
11675	}
11676	break;
11677	}
11678
11679	case VMX_EXIT_TRIPLE_FAULT:
11680	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11681	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11682	break;
11683	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11684	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11685	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11686	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11687	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11688
11689	/* Instruction specific VM-exits: */
11690	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11691	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11692	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11693	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11694	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11695	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11696	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11697	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11698	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11699	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11700	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11701	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11702	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11703	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11704	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11705	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11706	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11707	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11708	case VMX_EXIT_MOV_CRX:
11709	hmR0VmxReadExitQualVmcs(pVmxTransient);
11710	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11711	SET_BOTH(CRX_READ);
11712	else
11713	SET_BOTH(CRX_WRITE);
11714	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11715	break;
11716	case VMX_EXIT_MOV_DRX:
11717	hmR0VmxReadExitQualVmcs(pVmxTransient);
11718	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11719	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11720	SET_BOTH(DRX_READ);
11721	else
11722	SET_BOTH(DRX_WRITE);
11723	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11724	break;
11725	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11726	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11727	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11728	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11729	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11730	case VMX_EXIT_GDTR_IDTR_ACCESS:
11731	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11732	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11733	{
11734	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11735	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11736	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11737	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11738	}
11739	break;
11740
11741	case VMX_EXIT_LDTR_TR_ACCESS:
11742	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11743	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11744	{
11745	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11746	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11747	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11748	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11749	}
11750	break;
11751
11752	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11753	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11754	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11755	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11756	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11757	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11758	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11759	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11760	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11761	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11762	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11763
11764	/* Events that aren't relevant at this point. */
11765	case VMX_EXIT_EXT_INT:
11766	case VMX_EXIT_INT_WINDOW:
11767	case VMX_EXIT_NMI_WINDOW:
11768	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11769	case VMX_EXIT_PREEMPT_TIMER:
11770	case VMX_EXIT_IO_INSTR:
11771	break;
11772
11773	/* Errors and unexpected events. */
11774	case VMX_EXIT_INIT_SIGNAL:
11775	case VMX_EXIT_SIPI:
11776	case VMX_EXIT_IO_SMI:
11777	case VMX_EXIT_SMI:
11778	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11779	case VMX_EXIT_ERR_MSR_LOAD:
11780	case VMX_EXIT_ERR_MACHINE_CHECK:
11781	case VMX_EXIT_PML_FULL:
11782	case VMX_EXIT_VIRTUALIZED_EOI:
11783	break;
11784
11785	default:
11786	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11787	break;
11788	}
11789	#undef SET_BOTH
11790	#undef SET_EXIT
11791
11792	/*
11793	* Dtrace tracepoints go first. We do them here at once so we don't
11794	* have to copy the guest state saving and stuff a few dozen times.
11795	* Down side is that we've got to repeat the switch, though this time
11796	* we use enmEvent since the probes are a subset of what DBGF does.
11797	*/
11798	if (fDtrace1 \|\| fDtrace2)
11799	{
11800	hmR0VmxReadExitQualVmcs(pVmxTransient);
11801	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11802	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11803	switch (enmEvent1)
11804	{
11805	/** @todo consider which extra parameters would be helpful for each probe. */
11806	case DBGFEVENT_END: break;
11807	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11808	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11809	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11810	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11811	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11812	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11813	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11814	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11815	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11816	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11817	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11818	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11819	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11820	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11821	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11822	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11823	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11824	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11825	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11826	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11827	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11828	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11829	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11830	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11831	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11832	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11833	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11834	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11835	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11836	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11837	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11838	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11839	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11840	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11841	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11842	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11843	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11844	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11845	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11846	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11847	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11848	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11849	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11850	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11851	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11852	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11853	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11854	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11855	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11856	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11857	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11858	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11859	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11860	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11861	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11862	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11863	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11864	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11865	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11866	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11867	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11868	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11869	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11870	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11871	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11872	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11873	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11874	}
11875	switch (enmEvent2)
11876	{
11877	/** @todo consider which extra parameters would be helpful for each probe. */
11878	case DBGFEVENT_END: break;
11879	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11880	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11881	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11882	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11883	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11884	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11885	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11886	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11887	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11888	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11889	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11890	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11891	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11892	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11893	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11894	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11895	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11896	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11897	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11898	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11899	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11900	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11901	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11902	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11903	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11904	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11905	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11906	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11907	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11908	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11909	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11910	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11911	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11912	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11913	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11914	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11915	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11916	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11917	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11918	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11919	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11920	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11921	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11922	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11923	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11924	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11925	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11926	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11927	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11928	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11929	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11930	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11931	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11932	}
11933	}
11934
11935	/*
11936	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11937	* the DBGF call will do a full check).
11938	*
11939	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11940	* Note! If we have to events, we prioritize the first, i.e. the instruction
11941	* one, in order to avoid event nesting.
11942	*/
11943	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11944	if ( enmEvent1 != DBGFEVENT_END
11945	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11946	{
11947	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11948	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11949	if (rcStrict != VINF_SUCCESS)
11950	return rcStrict;
11951	}
11952	else if ( enmEvent2 != DBGFEVENT_END
11953	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11954	{
11955	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11956	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11957	if (rcStrict != VINF_SUCCESS)
11958	return rcStrict;
11959	}
11960
11961	return VINF_SUCCESS;
11962	}
11963
11964
11965	/**
11966	* Single-stepping VM-exit filtering.
11967	*
11968	* This is preprocessing the VM-exits and deciding whether we've gotten far
11969	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11970	* handling is performed.
11971	*
11972	* @returns Strict VBox status code (i.e. informational status codes too).
11973	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11974	* @param pVmxTransient The VMX-transient structure.
11975	* @param pDbgState The debug state.
11976	*/
11977	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11978	{
11979	/*
11980	* Expensive (saves context) generic dtrace VM-exit probe.
11981	*/
11982	uint32_t const uExitReason = pVmxTransient->uExitReason;
11983	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11984	{ /* more likely */ }
11985	else
11986	{
11987	hmR0VmxReadExitQualVmcs(pVmxTransient);
11988	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11989	AssertRC(rc);
11990	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11991	}
11992
11993	/*
11994	* Check for host NMI, just to get that out of the way.
11995	*/
11996	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11997	{ /* normally likely */ }
11998	else
11999	{
12000	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12001	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12002	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12003	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12004	}
12005
12006	/*
12007	* Check for single stepping event if we're stepping.
12008	*/
12009	if (pVCpu->hm.s.fSingleInstruction)
12010	{
12011	switch (uExitReason)
12012	{
12013	case VMX_EXIT_MTF:
12014	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12015
12016	/* Various events: */
12017	case VMX_EXIT_XCPT_OR_NMI:
12018	case VMX_EXIT_EXT_INT:
12019	case VMX_EXIT_TRIPLE_FAULT:
12020	case VMX_EXIT_INT_WINDOW:
12021	case VMX_EXIT_NMI_WINDOW:
12022	case VMX_EXIT_TASK_SWITCH:
12023	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12024	case VMX_EXIT_APIC_ACCESS:
12025	case VMX_EXIT_EPT_VIOLATION:
12026	case VMX_EXIT_EPT_MISCONFIG:
12027	case VMX_EXIT_PREEMPT_TIMER:
12028
12029	/* Instruction specific VM-exits: */
12030	case VMX_EXIT_CPUID:
12031	case VMX_EXIT_GETSEC:
12032	case VMX_EXIT_HLT:
12033	case VMX_EXIT_INVD:
12034	case VMX_EXIT_INVLPG:
12035	case VMX_EXIT_RDPMC:
12036	case VMX_EXIT_RDTSC:
12037	case VMX_EXIT_RSM:
12038	case VMX_EXIT_VMCALL:
12039	case VMX_EXIT_VMCLEAR:
12040	case VMX_EXIT_VMLAUNCH:
12041	case VMX_EXIT_VMPTRLD:
12042	case VMX_EXIT_VMPTRST:
12043	case VMX_EXIT_VMREAD:
12044	case VMX_EXIT_VMRESUME:
12045	case VMX_EXIT_VMWRITE:
12046	case VMX_EXIT_VMXOFF:
12047	case VMX_EXIT_VMXON:
12048	case VMX_EXIT_MOV_CRX:
12049	case VMX_EXIT_MOV_DRX:
12050	case VMX_EXIT_IO_INSTR:
12051	case VMX_EXIT_RDMSR:
12052	case VMX_EXIT_WRMSR:
12053	case VMX_EXIT_MWAIT:
12054	case VMX_EXIT_MONITOR:
12055	case VMX_EXIT_PAUSE:
12056	case VMX_EXIT_GDTR_IDTR_ACCESS:
12057	case VMX_EXIT_LDTR_TR_ACCESS:
12058	case VMX_EXIT_INVEPT:
12059	case VMX_EXIT_RDTSCP:
12060	case VMX_EXIT_INVVPID:
12061	case VMX_EXIT_WBINVD:
12062	case VMX_EXIT_XSETBV:
12063	case VMX_EXIT_RDRAND:
12064	case VMX_EXIT_INVPCID:
12065	case VMX_EXIT_VMFUNC:
12066	case VMX_EXIT_RDSEED:
12067	case VMX_EXIT_XSAVES:
12068	case VMX_EXIT_XRSTORS:
12069	{
12070	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12071	AssertRCReturn(rc, rc);
12072	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12073	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12074	return VINF_EM_DBG_STEPPED;
12075	break;
12076	}
12077
12078	/* Errors and unexpected events: */
12079	case VMX_EXIT_INIT_SIGNAL:
12080	case VMX_EXIT_SIPI:
12081	case VMX_EXIT_IO_SMI:
12082	case VMX_EXIT_SMI:
12083	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12084	case VMX_EXIT_ERR_MSR_LOAD:
12085	case VMX_EXIT_ERR_MACHINE_CHECK:
12086	case VMX_EXIT_PML_FULL:
12087	case VMX_EXIT_VIRTUALIZED_EOI:
12088	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12089	break;
12090
12091	default:
12092	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12093	break;
12094	}
12095	}
12096
12097	/*
12098	* Check for debugger event breakpoints and dtrace probes.
12099	*/
12100	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12101	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12102	{
12103	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12104	if (rcStrict != VINF_SUCCESS)
12105	return rcStrict;
12106	}
12107
12108	/*
12109	* Normal processing.
12110	*/
12111	#ifdef HMVMX_USE_FUNCTION_TABLE
12112	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12113	#else
12114	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12115	#endif
12116	}
12117
12118
12119	/**
12120	* Single steps guest code using hardware-assisted VMX.
12121	*
12122	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12123	* but single-stepping through the hypervisor debugger.
12124	*
12125	* @returns Strict VBox status code (i.e. informational status codes too).
12126	* @param pVCpu The cross context virtual CPU structure.
12127	* @param pcLoops Pointer to the number of executed loops.
12128	*
12129	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12130	*/
12131	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
12132	{
12133	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12134	Assert(pcLoops);
12135	Assert(*pcLoops <= cMaxResumeLoops);
12136
12137	VMXTRANSIENT VmxTransient;
12138	RT_ZERO(VmxTransient);
12139	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12140
12141	/* Set HMCPU indicators. */
12142	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12143	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12144	pVCpu->hm.s.fDebugWantRdTscExit = false;
12145	pVCpu->hm.s.fUsingDebugLoop = true;
12146
12147	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12148	VMXRUNDBGSTATE DbgState;
12149	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12150	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12151
12152	/*
12153	* The loop.
12154	*/
12155	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12156	for (;;)
12157	{
12158	Assert(!HMR0SuspendPending());
12159	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12160	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12161	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12162
12163	/* Set up VM-execution controls the next two can respond to. */
12164	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12165
12166	/*
12167	* Preparatory work for running guest code, this may force us to
12168	* return to ring-3.
12169	*
12170	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12171	*/
12172	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12173	if (rcStrict != VINF_SUCCESS)
12174	break;
12175
12176	/* Interrupts are disabled at this point! */
12177	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12178
12179	/* Override any obnoxious code in the above two calls. */
12180	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12181
12182	/*
12183	* Finally execute the guest.
12184	*/
12185	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12186
12187	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12188	/* Interrupts are re-enabled at this point! */
12189
12190	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12191	if (RT_SUCCESS(rcRun))
12192	{ /* very likely */ }
12193	else
12194	{
12195	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12196	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12197	return rcRun;
12198	}
12199
12200	/* Profile the VM-exit. */
12201	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12202	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12203	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12204	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12205	HMVMX_START_EXIT_DISPATCH_PROF();
12206
12207	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12208
12209	/*
12210	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12211	*/
12212	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12213	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12214	if (rcStrict != VINF_SUCCESS)
12215	break;
12216	if (++(*pcLoops) > cMaxResumeLoops)
12217	{
12218	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12219	rcStrict = VINF_EM_RAW_INTERRUPT;
12220	break;
12221	}
12222
12223	/*
12224	* Stepping: Did the RIP change, if so, consider it a single step.
12225	* Otherwise, make sure one of the TFs gets set.
12226	*/
12227	if (fStepping)
12228	{
12229	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12230	AssertRC(rc);
12231	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12232	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12233	{
12234	rcStrict = VINF_EM_DBG_STEPPED;
12235	break;
12236	}
12237	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12238	}
12239
12240	/*
12241	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12242	*/
12243	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12244	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12245	}
12246
12247	/*
12248	* Clear the X86_EFL_TF if necessary.
12249	*/
12250	if (pVCpu->hm.s.fClearTrapFlag)
12251	{
12252	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12253	AssertRC(rc);
12254	pVCpu->hm.s.fClearTrapFlag = false;
12255	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12256	}
12257	/** @todo there seems to be issues with the resume flag when the monitor trap
12258	* flag is pending without being used. Seen early in bios init when
12259	* accessing APIC page in protected mode. */
12260
12261	/*
12262	* Restore VM-exit control settings as we may not re-enter this function the
12263	* next time around.
12264	*/
12265	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12266
12267	/* Restore HMCPU indicators. */
12268	pVCpu->hm.s.fUsingDebugLoop = false;
12269	pVCpu->hm.s.fDebugWantRdTscExit = false;
12270	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12271
12272	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12273	return rcStrict;
12274	}
12275
12276
12277	/** @} */
12278
12279
12280	/**
12281	* Checks if any expensive dtrace probes are enabled and we should go to the
12282	* debug loop.
12283	*
12284	* @returns true if we should use debug loop, false if not.
12285	*/
12286	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12287	{
12288	/* It's probably faster to OR the raw 32-bit counter variables together.
12289	Since the variables are in an array and the probes are next to one
12290	another (more or less), we have good locality. So, better read
12291	eight-nine cache lines ever time and only have one conditional, than
12292	128+ conditionals, right? */
12293	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12294	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12295	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12296	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12297	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12298	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12299	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12300	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12301	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12302	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12303	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12304	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12305	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12306	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12307	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12308	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12309	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12310	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12311	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12312	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12313	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12314	) != 0
12315	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12316	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12317	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12318	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12319	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12320	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12321	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12322	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12323	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12324	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12325	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12326	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12327	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12328	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12329	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12330	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12331	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12332	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12333	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12334	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12335	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12336	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12337	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12338	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12339	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12340	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12341	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12342	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12343	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12344	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12345	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12346	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12347	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12348	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12349	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12350	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12351	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12352	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12353	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12354	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12355	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12356	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12357	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12358	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12359	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12360	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12361	) != 0
12362	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12363	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12364	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12365	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12366	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12367	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12368	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12369	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12370	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12371	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12372	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12373	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12374	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12375	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12376	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12377	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12378	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12379	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12380	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12381	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12382	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12383	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12384	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12385	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12386	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12387	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12388	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12389	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12390	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12391	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12392	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12393	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12394	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12395	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12396	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12397	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12398	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12399	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12400	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12401	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12402	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12403	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12404	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12405	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12406	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12407	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12408	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12409	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12410	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12411	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12412	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12413	) != 0;
12414	}
12415
12416
12417	/**
12418	* Runs the guest using hardware-assisted VMX.
12419	*
12420	* @returns Strict VBox status code (i.e. informational status codes too).
12421	* @param pVCpu The cross context virtual CPU structure.
12422	*/
12423	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPUCC pVCpu)
12424	{
12425	AssertPtr(pVCpu);
12426	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12427	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12428	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12429	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12430
12431	VBOXSTRICTRC rcStrict;
12432	uint32_t cLoops = 0;
12433	for (;;)
12434	{
12435	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12436	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12437	#else
12438	NOREF(pCtx);
12439	bool const fInNestedGuestMode = false;
12440	#endif
12441	if (!fInNestedGuestMode)
12442	{
12443	if ( !pVCpu->hm.s.fUseDebugLoop
12444	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12445	&& !DBGFIsStepping(pVCpu)
12446	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12447	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12448	else
12449	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12450	}
12451	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12452	else
12453	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12454
12455	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12456	{
12457	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12458	continue;
12459	}
12460	if (rcStrict == VINF_VMX_VMEXIT)
12461	{
12462	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12463	continue;
12464	}
12465	#endif
12466	break;
12467	}
12468
12469	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12470	switch (rcLoop)
12471	{
12472	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12473	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12474	}
12475
12476	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12477	if (RT_FAILURE(rc2))
12478	{
12479	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12480	rcStrict = rc2;
12481	}
12482	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12483	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12484	return rcStrict;
12485	}
12486
12487
12488	#ifndef HMVMX_USE_FUNCTION_TABLE
12489	/**
12490	* Handles a guest VM-exit from hardware-assisted VMX execution.
12491	*
12492	* @returns Strict VBox status code (i.e. informational status codes too).
12493	* @param pVCpu The cross context virtual CPU structure.
12494	* @param pVmxTransient The VMX-transient structure.
12495	*/
12496	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12497	{
12498	#ifdef DEBUG_ramshankar
12499	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12500	do { \
12501	if (a_fSave != 0) \
12502	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12503	VBOXSTRICTRC rcStrict = a_CallExpr; \
12504	if (a_fSave != 0) \
12505	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12506	return rcStrict; \
12507	} while (0)
12508	#else
12509	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12510	#endif
12511	uint32_t const uExitReason = pVmxTransient->uExitReason;
12512	switch (uExitReason)
12513	{
12514	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12515	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12516	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12517	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12518	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12519	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12520	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12521	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12522	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12523	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12524	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12525	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12526	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12527	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12528	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12529	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12530	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12531	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12532	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12533	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12534	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12535	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12536	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12537	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12538	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12539	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12540	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12541	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12542	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12543	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12544	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12545	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12546	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12547	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12548	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12549	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12550	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12551	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12552	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12553	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12554	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12555	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12556	#else
12557	case VMX_EXIT_VMCLEAR:
12558	case VMX_EXIT_VMLAUNCH:
12559	case VMX_EXIT_VMPTRLD:
12560	case VMX_EXIT_VMPTRST:
12561	case VMX_EXIT_VMREAD:
12562	case VMX_EXIT_VMRESUME:
12563	case VMX_EXIT_VMWRITE:
12564	case VMX_EXIT_VMXOFF:
12565	case VMX_EXIT_VMXON:
12566	case VMX_EXIT_INVVPID:
12567	case VMX_EXIT_INVEPT:
12568	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12569	#endif
12570
12571	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12572	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12573	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12574
12575	case VMX_EXIT_INIT_SIGNAL:
12576	case VMX_EXIT_SIPI:
12577	case VMX_EXIT_IO_SMI:
12578	case VMX_EXIT_SMI:
12579	case VMX_EXIT_ERR_MSR_LOAD:
12580	case VMX_EXIT_ERR_MACHINE_CHECK:
12581	case VMX_EXIT_PML_FULL:
12582	case VMX_EXIT_VIRTUALIZED_EOI:
12583	case VMX_EXIT_GDTR_IDTR_ACCESS:
12584	case VMX_EXIT_LDTR_TR_ACCESS:
12585	case VMX_EXIT_APIC_WRITE:
12586	case VMX_EXIT_RDRAND:
12587	case VMX_EXIT_RSM:
12588	case VMX_EXIT_VMFUNC:
12589	case VMX_EXIT_ENCLS:
12590	case VMX_EXIT_RDSEED:
12591	case VMX_EXIT_XSAVES:
12592	case VMX_EXIT_XRSTORS:
12593	case VMX_EXIT_UMWAIT:
12594	case VMX_EXIT_TPAUSE:
12595	default:
12596	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12597	}
12598	#undef VMEXIT_CALL_RET
12599	}
12600	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12601
12602
12603	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12604	/**
12605	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12606	*
12607	* @returns Strict VBox status code (i.e. informational status codes too).
12608	* @param pVCpu The cross context virtual CPU structure.
12609	* @param pVmxTransient The VMX-transient structure.
12610	*/
12611	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12612	{
12613	/** @todo NSTVMX: Remove after debugging page-fault issue. */
12614	#ifdef DEBUG_ramshankar
12615	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12616	Log4Func(("cs:rip=%#04x:%#RX64 rsp=%#RX64 eflags=%#RX32 cr3=%#RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12617	pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.cr3));
12618	#endif
12619
12620	uint32_t const uExitReason = pVmxTransient->uExitReason;
12621	switch (uExitReason)
12622	{
12623	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12624	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12625	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12626	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12627	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12628
12629	/*
12630	* We shouldn't direct host physical interrupts to the nested-guest.
12631	*/
12632	case VMX_EXIT_EXT_INT:
12633	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12634
12635	/*
12636	* Instructions that cause VM-exits unconditionally or the condition is
12637	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12638	* happens, it's guaranteed to be a nested-guest VM-exit).
12639	*
12640	* - Provides VM-exit instruction length ONLY.
12641	*/
12642	case VMX_EXIT_CPUID: /* Unconditional. */
12643	case VMX_EXIT_VMCALL:
12644	case VMX_EXIT_GETSEC:
12645	case VMX_EXIT_INVD:
12646	case VMX_EXIT_XSETBV:
12647	case VMX_EXIT_VMLAUNCH:
12648	case VMX_EXIT_VMRESUME:
12649	case VMX_EXIT_VMXOFF:
12650	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
12651	case VMX_EXIT_VMFUNC:
12652	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12653
12654	/*
12655	* Instructions that cause VM-exits unconditionally or the condition is
12656	* always is taken solely from the nested hypervisor (meaning if the VM-exit
12657	* happens, it's guaranteed to be a nested-guest VM-exit).
12658	*
12659	* - Provides VM-exit instruction length.
12660	* - Provides VM-exit information.
12661	* - Optionally provides Exit qualification.
12662	*
12663	* Since Exit qualification is 0 for all VM-exits where it is not
12664	* applicable, reading and passing it to the guest should produce
12665	* defined behavior.
12666	*
12667	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12668	*/
12669	case VMX_EXIT_INVEPT: /* Unconditional. */
12670	case VMX_EXIT_INVVPID:
12671	case VMX_EXIT_VMCLEAR:
12672	case VMX_EXIT_VMPTRLD:
12673	case VMX_EXIT_VMPTRST:
12674	case VMX_EXIT_VMXON:
12675	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
12676	case VMX_EXIT_LDTR_TR_ACCESS:
12677	case VMX_EXIT_RDRAND:
12678	case VMX_EXIT_RDSEED:
12679	case VMX_EXIT_XSAVES:
12680	case VMX_EXIT_XRSTORS:
12681	case VMX_EXIT_UMWAIT:
12682	case VMX_EXIT_TPAUSE:
12683	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12684
12685	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12686	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12687	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12688	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12689	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12690	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12691	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12692	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12693	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12694	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12695	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12696	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12697	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12698	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12699	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12700	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12701	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12702	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12703	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12704
12705	case VMX_EXIT_PREEMPT_TIMER:
12706	{
12707	/** @todo NSTVMX: Preempt timer. */
12708	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12709	}
12710
12711	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12712	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12713
12714	case VMX_EXIT_VMREAD:
12715	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12716
12717	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12718	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12719
12720	case VMX_EXIT_INIT_SIGNAL:
12721	case VMX_EXIT_SIPI:
12722	case VMX_EXIT_IO_SMI:
12723	case VMX_EXIT_SMI:
12724	case VMX_EXIT_ERR_MSR_LOAD:
12725	case VMX_EXIT_ERR_MACHINE_CHECK:
12726	case VMX_EXIT_PML_FULL:
12727	case VMX_EXIT_RSM:
12728	default:
12729	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12730	}
12731	}
12732	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12733
12734
12735	/** @name VM-exit helpers.
12736	* @{
12737	*/
12738	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12739	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12740	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12741
12742	/** Macro for VM-exits called unexpectedly. */
12743	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
12744	do { \
12745	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
12746	return VERR_VMX_UNEXPECTED_EXIT; \
12747	} while (0)
12748
12749	#ifdef VBOX_STRICT
12750	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12751	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12752	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12753
12754	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12755	do { \
12756	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12757	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12758	} while (0)
12759
12760	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12761	do { \
12762	AssertPtr((a_pVCpu)); \
12763	AssertPtr((a_pVmxTransient)); \
12764	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12765	Assert((a_pVmxTransient)->pVmcsInfo); \
12766	Assert(ASMIntAreEnabled()); \
12767	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12768	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12769	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
12770	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12771	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12772	HMVMX_ASSERT_PREEMPT_CPUID(); \
12773	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12774	} while (0)
12775
12776	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12777	do { \
12778	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12779	Assert((a_pVmxTransient)->fIsNestedGuest); \
12780	} while (0)
12781
12782	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12783	do { \
12784	Log4Func(("\n")); \
12785	} while (0)
12786	#else
12787	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12788	do { \
12789	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12790	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12791	} while (0)
12792
12793	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12794	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12795
12796	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12797	#endif
12798
12799	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12800	/** Macro that does the necessary privilege checks and intercepted VM-exits for
12801	* guests that attempted to execute a VMX instruction. */
12802	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
12803	do \
12804	{ \
12805	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
12806	if (rcStrictTmp == VINF_SUCCESS) \
12807	{ /* likely */ } \
12808	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12809	{ \
12810	Assert((a_pVCpu)->hm.s.Event.fPending); \
12811	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
12812	return VINF_SUCCESS; \
12813	} \
12814	else \
12815	{ \
12816	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
12817	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
12818	} \
12819	} while (0)
12820
12821	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
12822	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
12823	do \
12824	{ \
12825	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
12826	(a_pGCPtrEffAddr)); \
12827	if (rcStrictTmp == VINF_SUCCESS) \
12828	{ /* likely */ } \
12829	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
12830	{ \
12831	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
12832	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
12833	NOREF(uXcptTmp); \
12834	return VINF_SUCCESS; \
12835	} \
12836	else \
12837	{ \
12838	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
12839	return rcStrictTmp; \
12840	} \
12841	} while (0)
12842	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12843
12844
12845	/**
12846	* Advances the guest RIP by the specified number of bytes.
12847	*
12848	* @param pVCpu The cross context virtual CPU structure.
12849	* @param cbInstr Number of bytes to advance the RIP by.
12850	*
12851	* @remarks No-long-jump zone!!!
12852	*/
12853	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
12854	{
12855	/* Advance the RIP. */
12856	pVCpu->cpum.GstCtx.rip += cbInstr;
12857	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12858
12859	/* Update interrupt inhibition. */
12860	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12861	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12862	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12863	}
12864
12865
12866	/**
12867	* Advances the guest RIP after reading it from the VMCS.
12868	*
12869	* @returns VBox status code, no informational status codes.
12870	* @param pVCpu The cross context virtual CPU structure.
12871	* @param pVmxTransient The VMX-transient structure.
12872	*
12873	* @remarks No-long-jump zone!!!
12874	*/
12875	static int hmR0VmxAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12876	{
12877	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12878	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12879	AssertRCReturn(rc, rc);
12880
12881	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
12882	return VINF_SUCCESS;
12883	}
12884
12885
12886	/**
12887	* Handle a condition that occurred while delivering an event through the guest or
12888	* nested-guest IDT.
12889	*
12890	* @returns Strict VBox status code (i.e. informational status codes too).
12891	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12892	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12893	* to continue execution of the guest which will delivery the \#DF.
12894	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12895	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12896	*
12897	* @param pVCpu The cross context virtual CPU structure.
12898	* @param pVmxTransient The VMX-transient structure.
12899	*
12900	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
12901	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
12902	* is due to an EPT violation, PML full or SPP-related event.
12903	*
12904	* @remarks No-long-jump zone!!!
12905	*/
12906	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
12907	{
12908	Assert(!pVCpu->hm.s.Event.fPending);
12909	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
12910	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
12911	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
12912	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
12913	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
12914
12915	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12916	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12917	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
12918	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12919	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
12920	{
12921	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
12922	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
12923
12924	/*
12925	* If the event was a software interrupt (generated with INT n) or a software exception
12926	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12927	* can handle the VM-exit and continue guest execution which will re-execute the
12928	* instruction rather than re-injecting the exception, as that can cause premature
12929	* trips to ring-3 before injection and involve TRPM which currently has no way of
12930	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12931	* the problem).
12932	*/
12933	IEMXCPTRAISE enmRaise;
12934	IEMXCPTRAISEINFO fRaiseInfo;
12935	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12936	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12937	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12938	{
12939	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12940	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12941	}
12942	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
12943	{
12944	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
12945	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12946	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
12947
12948	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12949	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12950
12951	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12952
12953	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12954	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12955	{
12956	pVmxTransient->fVectoringPF = true;
12957	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12958	}
12959	}
12960	else
12961	{
12962	/*
12963	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12964	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12965	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12966	*/
12967	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12968	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12969	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12970	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12971	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12972	}
12973
12974	/*
12975	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12976	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12977	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12978	* subsequent VM-entry would fail, see @bugref{7445}.
12979	*
12980	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
12981	*/
12982	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12983	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
12984	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
12985	&& CPUMIsGuestNmiBlocking(pVCpu))
12986	{
12987	CPUMSetGuestNmiBlocking(pVCpu, false);
12988	}
12989
12990	switch (enmRaise)
12991	{
12992	case IEMXCPTRAISE_CURRENT_XCPT:
12993	{
12994	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
12995	Assert(rcStrict == VINF_SUCCESS);
12996	break;
12997	}
12998
12999	case IEMXCPTRAISE_PREV_EVENT:
13000	{
13001	uint32_t u32ErrCode;
13002	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13003	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13004	else
13005	u32ErrCode = 0;
13006
13007	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13008	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13009	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13010	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13011
13012	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13013	pVCpu->hm.s.Event.u32ErrCode));
13014	Assert(rcStrict == VINF_SUCCESS);
13015	break;
13016	}
13017
13018	case IEMXCPTRAISE_REEXEC_INSTR:
13019	Assert(rcStrict == VINF_SUCCESS);
13020	break;
13021
13022	case IEMXCPTRAISE_DOUBLE_FAULT:
13023	{
13024	/*
13025	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13026	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13027	*/
13028	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13029	{
13030	pVmxTransient->fVectoringDoublePF = true;
13031	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13032	pVCpu->cpum.GstCtx.cr2));
13033	rcStrict = VINF_SUCCESS;
13034	}
13035	else
13036	{
13037	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13038	hmR0VmxSetPendingXcptDF(pVCpu);
13039	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13040	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13041	rcStrict = VINF_HM_DOUBLE_FAULT;
13042	}
13043	break;
13044	}
13045
13046	case IEMXCPTRAISE_TRIPLE_FAULT:
13047	{
13048	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13049	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13050	rcStrict = VINF_EM_RESET;
13051	break;
13052	}
13053
13054	case IEMXCPTRAISE_CPU_HANG:
13055	{
13056	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13057	rcStrict = VERR_EM_GUEST_CPU_HANG;
13058	break;
13059	}
13060
13061	default:
13062	{
13063	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13064	rcStrict = VERR_VMX_IPE_2;
13065	break;
13066	}
13067	}
13068	}
13069	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13070	&& !CPUMIsGuestNmiBlocking(pVCpu))
13071	{
13072	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13073	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13074	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13075	{
13076	/*
13077	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13078	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13079	* that NMIs remain blocked until the IRET execution is completed.
13080	*
13081	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13082	*/
13083	CPUMSetGuestNmiBlocking(pVCpu, true);
13084	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13085	}
13086	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13087	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13088	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13089	{
13090	/*
13091	* Execution of IRET caused an EPT violation, page-modification log-full event or
13092	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13093	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13094	* that NMIs remain blocked until the IRET execution is completed.
13095	*
13096	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13097	*/
13098	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13099	{
13100	CPUMSetGuestNmiBlocking(pVCpu, true);
13101	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13102	}
13103	}
13104	}
13105
13106	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13107	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13108	return rcStrict;
13109	}
13110
13111
13112	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13113	/**
13114	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13115	* guest attempting to execute a VMX instruction.
13116	*
13117	* @returns Strict VBox status code (i.e. informational status codes too).
13118	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13119	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13120	*
13121	* @param pVCpu The cross context virtual CPU structure.
13122	* @param uExitReason The VM-exit reason.
13123	*
13124	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13125	* @remarks No-long-jump zone!!!
13126	*/
13127	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
13128	{
13129	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13130	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13131
13132	/*
13133	* The physical CPU would have already checked the CPU mode/code segment.
13134	* We shall just assert here for paranoia.
13135	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13136	*/
13137	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13138	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13139	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13140
13141	if (uExitReason == VMX_EXIT_VMXON)
13142	{
13143	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13144
13145	/*
13146	* We check CR4.VMXE because it is required to be always set while in VMX operation
13147	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13148	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13149	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13150	*/
13151	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13152	{
13153	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13154	hmR0VmxSetPendingXcptUD(pVCpu);
13155	return VINF_HM_PENDING_XCPT;
13156	}
13157	}
13158	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13159	{
13160	/*
13161	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13162	* (other than VMXON), we need to raise a #UD.
13163	*/
13164	Log4Func(("Not in VMX root mode -> #UD\n"));
13165	hmR0VmxSetPendingXcptUD(pVCpu);
13166	return VINF_HM_PENDING_XCPT;
13167	}
13168
13169	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13170	return VINF_SUCCESS;
13171	}
13172
13173
13174	/**
13175	* Decodes the memory operand of an instruction that caused a VM-exit.
13176	*
13177	* The Exit qualification field provides the displacement field for memory
13178	* operand instructions, if any.
13179	*
13180	* @returns Strict VBox status code (i.e. informational status codes too).
13181	* @retval VINF_SUCCESS if the operand was successfully decoded.
13182	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13183	* operand.
13184	* @param pVCpu The cross context virtual CPU structure.
13185	* @param uExitInstrInfo The VM-exit instruction information field.
13186	* @param enmMemAccess The memory operand's access type (read or write).
13187	* @param GCPtrDisp The instruction displacement field, if any. For
13188	* RIP-relative addressing pass RIP + displacement here.
13189	* @param pGCPtrMem Where to store the effective destination memory address.
13190	*
13191	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13192	* virtual-8086 mode hence skips those checks while verifying if the
13193	* segment is valid.
13194	*/
13195	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13196	PRTGCPTR pGCPtrMem)
13197	{
13198	Assert(pGCPtrMem);
13199	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13200	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13201	\| CPUMCTX_EXTRN_CR0);
13202
13203	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13204	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13205	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13206
13207	VMXEXITINSTRINFO ExitInstrInfo;
13208	ExitInstrInfo.u = uExitInstrInfo;
13209	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13210	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13211	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13212	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13213	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13214	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13215	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13216	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13217	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13218
13219	/*
13220	* Validate instruction information.
13221	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13222	*/
13223	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13224	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13225	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13226	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13227	AssertLogRelMsgReturn(fIsMemOperand,
13228	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13229
13230	/*
13231	* Compute the complete effective address.
13232	*
13233	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13234	* See AMD spec. 4.5.2 "Segment Registers".
13235	*/
13236	RTGCPTR GCPtrMem = GCPtrDisp;
13237	if (fBaseRegValid)
13238	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13239	if (fIdxRegValid)
13240	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13241
13242	RTGCPTR const GCPtrOff = GCPtrMem;
13243	if ( !fIsLongMode
13244	\|\| iSegReg >= X86_SREG_FS)
13245	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13246	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13247
13248	/*
13249	* Validate effective address.
13250	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13251	*/
13252	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13253	Assert(cbAccess > 0);
13254	if (fIsLongMode)
13255	{
13256	if (X86_IS_CANONICAL(GCPtrMem))
13257	{
13258	*pGCPtrMem = GCPtrMem;
13259	return VINF_SUCCESS;
13260	}
13261
13262	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13263	* "Data Limit Checks in 64-bit Mode". */
13264	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13265	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13266	return VINF_HM_PENDING_XCPT;
13267	}
13268
13269	/*
13270	* This is a watered down version of iemMemApplySegment().
13271	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13272	* and segment CPL/DPL checks are skipped.
13273	*/
13274	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13275	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13276	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13277
13278	/* Check if the segment is present and usable. */
13279	if ( pSel->Attr.n.u1Present
13280	&& !pSel->Attr.n.u1Unusable)
13281	{
13282	Assert(pSel->Attr.n.u1DescType);
13283	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13284	{
13285	/* Check permissions for the data segment. */
13286	if ( enmMemAccess == VMXMEMACCESS_WRITE
13287	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13288	{
13289	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13290	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13291	return VINF_HM_PENDING_XCPT;
13292	}
13293
13294	/* Check limits if it's a normal data segment. */
13295	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13296	{
13297	if ( GCPtrFirst32 > pSel->u32Limit
13298	\|\| GCPtrLast32 > pSel->u32Limit)
13299	{
13300	Log4Func(("Data segment limit exceeded. "
13301	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13302	GCPtrLast32, pSel->u32Limit));
13303	if (iSegReg == X86_SREG_SS)
13304	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13305	else
13306	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13307	return VINF_HM_PENDING_XCPT;
13308	}
13309	}
13310	else
13311	{
13312	/* Check limits if it's an expand-down data segment.
13313	Note! The upper boundary is defined by the B bit, not the G bit! */
13314	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13315	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13316	{
13317	Log4Func(("Expand-down data segment limit exceeded. "
13318	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13319	GCPtrLast32, pSel->u32Limit));
13320	if (iSegReg == X86_SREG_SS)
13321	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13322	else
13323	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13324	return VINF_HM_PENDING_XCPT;
13325	}
13326	}
13327	}
13328	else
13329	{
13330	/* Check permissions for the code segment. */
13331	if ( enmMemAccess == VMXMEMACCESS_WRITE
13332	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13333	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13334	{
13335	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13336	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13337	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13338	return VINF_HM_PENDING_XCPT;
13339	}
13340
13341	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13342	if ( GCPtrFirst32 > pSel->u32Limit
13343	\|\| GCPtrLast32 > pSel->u32Limit)
13344	{
13345	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13346	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13347	if (iSegReg == X86_SREG_SS)
13348	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13349	else
13350	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13351	return VINF_HM_PENDING_XCPT;
13352	}
13353	}
13354	}
13355	else
13356	{
13357	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13358	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13359	return VINF_HM_PENDING_XCPT;
13360	}
13361
13362	*pGCPtrMem = GCPtrMem;
13363	return VINF_SUCCESS;
13364	}
13365	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13366
13367
13368	/**
13369	* VM-exit helper for LMSW.
13370	*/
13371	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13372	{
13373	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13374	AssertRCReturn(rc, rc);
13375
13376	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13377	AssertMsg( rcStrict == VINF_SUCCESS
13378	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13379
13380	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13381	if (rcStrict == VINF_IEM_RAISED_XCPT)
13382	{
13383	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13384	rcStrict = VINF_SUCCESS;
13385	}
13386
13387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13388	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13389	return rcStrict;
13390	}
13391
13392
13393	/**
13394	* VM-exit helper for CLTS.
13395	*/
13396	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13397	{
13398	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13399	AssertRCReturn(rc, rc);
13400
13401	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13402	AssertMsg( rcStrict == VINF_SUCCESS
13403	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13404
13405	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13406	if (rcStrict == VINF_IEM_RAISED_XCPT)
13407	{
13408	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13409	rcStrict = VINF_SUCCESS;
13410	}
13411
13412	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13413	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13414	return rcStrict;
13415	}
13416
13417
13418	/**
13419	* VM-exit helper for MOV from CRx (CRx read).
13420	*/
13421	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13422	{
13423	Assert(iCrReg < 16);
13424	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13425
13426	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13427	AssertRCReturn(rc, rc);
13428
13429	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13430	AssertMsg( rcStrict == VINF_SUCCESS
13431	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13432
13433	if (iGReg == X86_GREG_xSP)
13434	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13435	else
13436	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13437	#ifdef VBOX_WITH_STATISTICS
13438	switch (iCrReg)
13439	{
13440	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13441	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13442	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13443	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13444	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13445	}
13446	#endif
13447	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13448	return rcStrict;
13449	}
13450
13451
13452	/**
13453	* VM-exit helper for MOV to CRx (CRx write).
13454	*/
13455	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13456	{
13457	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13458	AssertRCReturn(rc, rc);
13459
13460	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13461	AssertMsg( rcStrict == VINF_SUCCESS
13462	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13463	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13464
13465	switch (iCrReg)
13466	{
13467	case 0:
13468	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
13469	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13470	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13471	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13472	break;
13473
13474	case 2:
13475	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13476	/* Nothing to do here, CR2 it's not part of the VMCS. */
13477	break;
13478
13479	case 3:
13480	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13481	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13482	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13483	break;
13484
13485	case 4:
13486	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13487	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13488	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13489	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13490	break;
13491
13492	case 8:
13493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13494	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13496	break;
13497
13498	default:
13499	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13500	break;
13501	}
13502
13503	if (rcStrict == VINF_IEM_RAISED_XCPT)
13504	{
13505	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13506	rcStrict = VINF_SUCCESS;
13507	}
13508	return rcStrict;
13509	}
13510
13511
13512	/**
13513	* VM-exit exception handler for \#PF (Page-fault exception).
13514	*
13515	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13516	*/
13517	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13518	{
13519	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13520	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
13521	hmR0VmxReadExitQualVmcs(pVmxTransient);
13522
13523	if (!pVM->hm.s.fNestedPaging)
13524	{ /* likely */ }
13525	else
13526	{
13527	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13528	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13529	#endif
13530	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13531	if (!pVmxTransient->fVectoringDoublePF)
13532	{
13533	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13534	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13535	}
13536	else
13537	{
13538	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13539	Assert(!pVmxTransient->fIsNestedGuest);
13540	hmR0VmxSetPendingXcptDF(pVCpu);
13541	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13542	}
13543	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13544	return VINF_SUCCESS;
13545	}
13546
13547	Assert(!pVmxTransient->fIsNestedGuest);
13548
13549	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13550	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13551	if (pVmxTransient->fVectoringPF)
13552	{
13553	Assert(pVCpu->hm.s.Event.fPending);
13554	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13555	}
13556
13557	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13558	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13559	AssertRCReturn(rc, rc);
13560
13561	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
13562	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
13563
13564	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13565	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13566
13567	Log4Func(("#PF: rc=%Rrc\n", rc));
13568	if (rc == VINF_SUCCESS)
13569	{
13570	/*
13571	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13572	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13573	*/
13574	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13575	TRPMResetTrap(pVCpu);
13576	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13577	return rc;
13578	}
13579
13580	if (rc == VINF_EM_RAW_GUEST_TRAP)
13581	{
13582	if (!pVmxTransient->fVectoringDoublePF)
13583	{
13584	/* It's a guest page fault and needs to be reflected to the guest. */
13585	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13586	TRPMResetTrap(pVCpu);
13587	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13588	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13589	uGstErrorCode, pVmxTransient->uExitQual);
13590	}
13591	else
13592	{
13593	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13594	TRPMResetTrap(pVCpu);
13595	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13596	hmR0VmxSetPendingXcptDF(pVCpu);
13597	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13598	}
13599
13600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13601	return VINF_SUCCESS;
13602	}
13603
13604	TRPMResetTrap(pVCpu);
13605	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13606	return rc;
13607	}
13608
13609
13610	/**
13611	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13612	*
13613	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13614	*/
13615	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13616	{
13617	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13618	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13619
13620	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13621	AssertRCReturn(rc, rc);
13622
13623	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13624	{
13625	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13626	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13627
13628	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13629	* provides VM-exit instruction length. If this causes problem later,
13630	* disassemble the instruction like it's done on AMD-V. */
13631	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13632	AssertRCReturn(rc2, rc2);
13633	return rc;
13634	}
13635
13636	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
13637	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13638	return VINF_SUCCESS;
13639	}
13640
13641
13642	/**
13643	* VM-exit exception handler for \#BP (Breakpoint exception).
13644	*
13645	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13646	*/
13647	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13648	{
13649	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13650	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13651
13652	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13653	AssertRCReturn(rc, rc);
13654
13655	if (!pVmxTransient->fIsNestedGuest)
13656	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13657	else
13658	rc = VINF_EM_RAW_GUEST_TRAP;
13659
13660	if (rc == VINF_EM_RAW_GUEST_TRAP)
13661	{
13662	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13663	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13664	rc = VINF_SUCCESS;
13665	}
13666
13667	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
13668	return rc;
13669	}
13670
13671
13672	/**
13673	* VM-exit exception handler for \#AC (Alignment-check exception).
13674	*
13675	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13676	*/
13677	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13678	{
13679	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13680	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13681
13682	/* Re-inject it. We'll detect any nesting before getting here. */
13683	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13684	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13685	return VINF_SUCCESS;
13686	}
13687
13688
13689	/**
13690	* VM-exit exception handler for \#DB (Debug exception).
13691	*
13692	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13693	*/
13694	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13695	{
13696	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13697	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13698
13699	/*
13700	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13701	*/
13702	hmR0VmxReadExitQualVmcs(pVmxTransient);
13703
13704	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13705	uint64_t const uDR6 = X86_DR6_INIT_VAL
13706	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13707	\| X86_DR6_BD \| X86_DR6_BS));
13708
13709	int rc;
13710	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13711	if (!pVmxTransient->fIsNestedGuest)
13712	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13713	else
13714	rc = VINF_EM_RAW_GUEST_TRAP;
13715	Log6Func(("rc=%Rrc\n", rc));
13716	if (rc == VINF_EM_RAW_GUEST_TRAP)
13717	{
13718	/*
13719	* The exception was for the guest. Update DR6, DR7.GD and
13720	* IA32_DEBUGCTL.LBR before forwarding it.
13721	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
13722	*/
13723	VMMRZCallRing3Disable(pVCpu);
13724	HM_DISABLE_PREEMPT(pVCpu);
13725
13726	pCtx->dr[6] &= ~X86_DR6_B_MASK;
13727	pCtx->dr[6] \|= uDR6;
13728	if (CPUMIsGuestDebugStateActive(pVCpu))
13729	ASMSetDR6(pCtx->dr[6]);
13730
13731	HM_RESTORE_PREEMPT();
13732	VMMRZCallRing3Enable(pVCpu);
13733
13734	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
13735	AssertRCReturn(rc, rc);
13736
13737	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13738	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
13739
13740	/* Paranoia. */
13741	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
13742	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
13743
13744	rc = VMXWriteVmcsNw(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
13745	AssertRC(rc);
13746
13747	/*
13748	* Raise #DB in the guest.
13749	*
13750	* It is important to reflect exactly what the VM-exit gave us (preserving the
13751	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13752	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13753	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13754	*
13755	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13756	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13757	*/
13758	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13759	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13760	return VINF_SUCCESS;
13761	}
13762
13763	/*
13764	* Not a guest trap, must be a hypervisor related debug event then.
13765	* Update DR6 in case someone is interested in it.
13766	*/
13767	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13768	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13769	CPUMSetHyperDR6(pVCpu, uDR6);
13770
13771	return rc;
13772	}
13773
13774
13775	/**
13776	* Hacks its way around the lovely mesa driver's backdoor accesses.
13777	*
13778	* @sa hmR0SvmHandleMesaDrvGp.
13779	*/
13780	static int hmR0VmxHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13781	{
13782	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
13783	RT_NOREF(pCtx);
13784
13785	/* For now we'll just skip the instruction. */
13786	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13787	}
13788
13789
13790	/**
13791	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
13792	* backdoor logging w/o checking what it is running inside.
13793	*
13794	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
13795	* backdoor port and magic numbers loaded in registers.
13796	*
13797	* @returns true if it is, false if it isn't.
13798	* @sa hmR0SvmIsMesaDrvGp.
13799	*/
13800	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
13801	{
13802	/* 0xed: IN eAX,dx */
13803	uint8_t abInstr[1];
13804	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
13805	return false;
13806
13807	/* Check that it is #GP(0). */
13808	if (pVmxTransient->uExitIntErrorCode != 0)
13809	return false;
13810
13811	/* Check magic and port. */
13812	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
13813	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
13814	if (pCtx->rax != UINT32_C(0x564d5868))
13815	return false;
13816	if (pCtx->dx != UINT32_C(0x5658))
13817	return false;
13818
13819	/* Flat ring-3 CS. */
13820	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
13821	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
13822	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
13823	if (pCtx->cs.Attr.n.u2Dpl != 3)
13824	return false;
13825	if (pCtx->cs.u64Base != 0)
13826	return false;
13827
13828	/* Check opcode. */
13829	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
13830	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
13831	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
13832	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
13833	if (RT_FAILURE(rc))
13834	return false;
13835	if (abInstr[0] != 0xed)
13836	return false;
13837
13838	return true;
13839	}
13840
13841
13842	/**
13843	* VM-exit exception handler for \#GP (General-protection exception).
13844	*
13845	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13846	*/
13847	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13848	{
13849	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13850	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13851
13852	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13853	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13854	if (pVmcsInfo->RealMode.fRealOnV86Active)
13855	{ /* likely */ }
13856	else
13857	{
13858	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13859	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
13860	#endif
13861	/*
13862	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
13863	* executing a nested-guest, reflect #GP to the guest or nested-guest.
13864	*/
13865	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13866	AssertRCReturn(rc, rc);
13867	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13868	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13869
13870	if ( pVmxTransient->fIsNestedGuest
13871	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
13872	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
13873	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13874	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13875	else
13876	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
13877	return rc;
13878	}
13879
13880	Assert(CPUMIsGuestInRealModeEx(pCtx));
13881	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13882	Assert(!pVmxTransient->fIsNestedGuest);
13883
13884	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13885	AssertRCReturn(rc, rc);
13886
13887	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13888	if (rcStrict == VINF_SUCCESS)
13889	{
13890	if (!CPUMIsGuestInRealModeEx(pCtx))
13891	{
13892	/*
13893	* The guest is no longer in real-mode, check if we can continue executing the
13894	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13895	*/
13896	pVmcsInfo->RealMode.fRealOnV86Active = false;
13897	if (HMCanExecuteVmxGuest(pVCpu->pVMR0, pVCpu, pCtx))
13898	{
13899	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
13900	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13901	}
13902	else
13903	{
13904	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13905	rcStrict = VINF_EM_RESCHEDULE;
13906	}
13907	}
13908	else
13909	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13910	}
13911	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13912	{
13913	rcStrict = VINF_SUCCESS;
13914	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13915	}
13916	return VBOXSTRICTRC_VAL(rcStrict);
13917	}
13918
13919
13920	/**
13921	* VM-exit exception handler wrapper for all other exceptions that are not handled
13922	* by a specific handler.
13923	*
13924	* This simply re-injects the exception back into the VM without any special
13925	* processing.
13926	*
13927	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13928	*/
13929	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13930	{
13931	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13932
13933	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13934	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13935	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
13936	("uVector=%#x u32XcptBitmap=%#X32\n",
13937	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
13938	NOREF(pVmcsInfo);
13939	#endif
13940
13941	/*
13942	* Re-inject the exception into the guest. This cannot be a double-fault condition which
13943	* would have been handled while checking exits due to event delivery.
13944	*/
13945	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13946
13947	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13948	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13949	AssertRCReturn(rc, rc);
13950	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13951	#endif
13952
13953	#ifdef VBOX_WITH_STATISTICS
13954	switch (uVector)
13955	{
13956	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
13957	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
13958	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
13959	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
13960	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
13961	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
13962	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
13963	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
13964	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
13965	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
13966	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
13967	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
13968	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
13969	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
13970	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
13971	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
13972	default:
13973	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13974	break;
13975	}
13976	#endif
13977
13978	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
13979	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
13980	NOREF(uVector);
13981
13982	/* Re-inject the original exception into the guest. */
13983	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13984	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13985	return VINF_SUCCESS;
13986	}
13987
13988
13989	/**
13990	* VM-exit exception handler for all exceptions (except NMIs!).
13991	*
13992	* @remarks This may be called for both guests and nested-guests. Take care to not
13993	* make assumptions and avoid doing anything that is not relevant when
13994	* executing a nested-guest (e.g., Mesa driver hacks).
13995	*/
13996	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
13997	{
13998	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
13999
14000	/*
14001	* If this VM-exit occurred while delivering an event through the guest IDT, take
14002	* action based on the return code and additional hints (e.g. for page-faults)
14003	* that will be updated in the VMX transient structure.
14004	*/
14005	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14006	if (rcStrict == VINF_SUCCESS)
14007	{
14008	/*
14009	* If an exception caused a VM-exit due to delivery of an event, the original
14010	* event may have to be re-injected into the guest. We shall reinject it and
14011	* continue guest execution. However, page-fault is a complicated case and
14012	* needs additional processing done in hmR0VmxExitXcptPF().
14013	*/
14014	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14015	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14016	if ( !pVCpu->hm.s.Event.fPending
14017	\|\| uVector == X86_XCPT_PF)
14018	{
14019	switch (uVector)
14020	{
14021	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
14022	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
14023	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
14024	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
14025	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
14026	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
14027	default:
14028	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
14029	}
14030	}
14031	/* else: inject pending event before resuming guest execution. */
14032	}
14033	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
14034	{
14035	Assert(pVCpu->hm.s.Event.fPending);
14036	rcStrict = VINF_SUCCESS;
14037	}
14038
14039	return rcStrict;
14040	}
14041	/** @} */
14042
14043
14044	/** @name VM-exit handlers.
14045	* @{
14046	*/
14047	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14048	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14049	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14050
14051	/**
14052	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14053	*/
14054	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14055	{
14056	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14057	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14058	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14059	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14060	return VINF_SUCCESS;
14061	return VINF_EM_RAW_INTERRUPT;
14062	}
14063
14064
14065	/**
14066	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14067	* VM-exit.
14068	*/
14069	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14070	{
14071	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14072	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14073
14074	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14075
14076	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14077	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14078	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14079
14080	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14081	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14082	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14083	NOREF(pVmcsInfo);
14084
14085	VBOXSTRICTRC rcStrict;
14086	switch (uExitIntType)
14087	{
14088	/*
14089	* Host physical NMIs:
14090	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14091	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14092	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14093	*
14094	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14095	* See Intel spec. 27.5.5 "Updating Non-Register State".
14096	*/
14097	case VMX_EXIT_INT_INFO_TYPE_NMI:
14098	{
14099	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14100	break;
14101	}
14102
14103	/*
14104	* Privileged software exceptions (#DB from ICEBP),
14105	* Software exceptions (#BP and #OF),
14106	* Hardware exceptions:
14107	* Process the required exceptions and resume guest execution if possible.
14108	*/
14109	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14110	Assert(uVector == X86_XCPT_DB);
14111	RT_FALL_THRU();
14112	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14113	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14114	RT_FALL_THRU();
14115	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14116	{
14117	NOREF(uVector);
14118	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14119	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14120	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14121	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14122
14123	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14124	break;
14125	}
14126
14127	default:
14128	{
14129	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14130	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14131	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14132	break;
14133	}
14134	}
14135
14136	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14137	return rcStrict;
14138	}
14139
14140
14141	/**
14142	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14143	*/
14144	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14145	{
14146	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14147
14148	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14149	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14150	hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14151
14152	/* Evaluate and deliver pending events and resume guest execution. */
14153	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14154	return VINF_SUCCESS;
14155	}
14156
14157
14158	/**
14159	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14160	*/
14161	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14162	{
14163	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14164
14165	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14166	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14167	{
14168	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14169	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14170	}
14171
14172	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14173
14174	/*
14175	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14176	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14177	*/
14178	uint32_t fIntrState;
14179	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14180	AssertRC(rc);
14181	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14182	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14183	{
14184	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14185	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14186
14187	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14188	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14189	AssertRC(rc);
14190	}
14191
14192	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14193	hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14194
14195	/* Evaluate and deliver pending events and resume guest execution. */
14196	return VINF_SUCCESS;
14197	}
14198
14199
14200	/**
14201	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14202	*/
14203	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14204	{
14205	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14206	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14207	}
14208
14209
14210	/**
14211	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14212	*/
14213	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14214	{
14215	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14216	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14217	}
14218
14219
14220	/**
14221	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14222	*/
14223	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14224	{
14225	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14226
14227	/*
14228	* Get the state we need and update the exit history entry.
14229	*/
14230	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14231	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14232
14233	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14234	AssertRCReturn(rc, rc);
14235
14236	VBOXSTRICTRC rcStrict;
14237	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14238	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14239	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14240	if (!pExitRec)
14241	{
14242	/*
14243	* Regular CPUID instruction execution.
14244	*/
14245	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
14246	if (rcStrict == VINF_SUCCESS)
14247	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14248	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14249	{
14250	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14251	rcStrict = VINF_SUCCESS;
14252	}
14253	}
14254	else
14255	{
14256	/*
14257	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14258	*/
14259	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14260	AssertRCReturn(rc2, rc2);
14261
14262	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14263	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14264
14265	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14266	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14267
14268	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14269	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14270	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14271	}
14272	return rcStrict;
14273	}
14274
14275
14276	/**
14277	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14278	*/
14279	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14280	{
14281	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14282
14283	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14284	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14285	AssertRCReturn(rc, rc);
14286
14287	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14288	return VINF_EM_RAW_EMULATE_INSTR;
14289
14290	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14291	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14292	}
14293
14294
14295	/**
14296	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14297	*/
14298	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14299	{
14300	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14301
14302	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14303	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14304	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14305	AssertRCReturn(rc, rc);
14306
14307	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
14308	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14309	{
14310	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14311	we must reset offsetting on VM-entry. See @bugref{6634}. */
14312	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14313	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14315	}
14316	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14317	{
14318	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14319	rcStrict = VINF_SUCCESS;
14320	}
14321	return rcStrict;
14322	}
14323
14324
14325	/**
14326	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14327	*/
14328	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14329	{
14330	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14331
14332	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14333	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14334	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14335	AssertRCReturn(rc, rc);
14336
14337	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
14338	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14339	{
14340	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14341	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14342	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14343	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14344	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14345	}
14346	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14347	{
14348	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14349	rcStrict = VINF_SUCCESS;
14350	}
14351	return rcStrict;
14352	}
14353
14354
14355	/**
14356	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14357	*/
14358	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14359	{
14360	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14361
14362	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14363	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14364	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14365	AssertRCReturn(rc, rc);
14366
14367	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14368	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14369	if (RT_LIKELY(rc == VINF_SUCCESS))
14370	{
14371	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14372	Assert(pVmxTransient->cbExitInstr == 2);
14373	}
14374	else
14375	{
14376	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14377	rc = VERR_EM_INTERPRETER;
14378	}
14379	return rc;
14380	}
14381
14382
14383	/**
14384	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14385	*/
14386	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14387	{
14388	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14389
14390	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14391	if (EMAreHypercallInstructionsEnabled(pVCpu))
14392	{
14393	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14394	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14395	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14396	AssertRCReturn(rc, rc);
14397
14398	/* Perform the hypercall. */
14399	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14400	if (rcStrict == VINF_SUCCESS)
14401	{
14402	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14403	AssertRCReturn(rc, rc);
14404	}
14405	else
14406	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14407	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14408	\|\| RT_FAILURE(rcStrict));
14409
14410	/* If the hypercall changes anything other than guest's general-purpose registers,
14411	we would need to reload the guest changed bits here before VM-entry. */
14412	}
14413	else
14414	Log4Func(("Hypercalls not enabled\n"));
14415
14416	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14417	if (RT_FAILURE(rcStrict))
14418	{
14419	hmR0VmxSetPendingXcptUD(pVCpu);
14420	rcStrict = VINF_SUCCESS;
14421	}
14422
14423	return rcStrict;
14424	}
14425
14426
14427	/**
14428	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14429	*/
14430	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14431	{
14432	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14433	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14434
14435	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14436	hmR0VmxReadExitQualVmcs(pVmxTransient);
14437	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14438	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14439	AssertRCReturn(rc, rc);
14440
14441	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
14442
14443	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14444	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14445	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14446	{
14447	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14448	rcStrict = VINF_SUCCESS;
14449	}
14450	else
14451	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14452	VBOXSTRICTRC_VAL(rcStrict)));
14453	return rcStrict;
14454	}
14455
14456
14457	/**
14458	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14459	*/
14460	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14461	{
14462	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14463
14464	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14465	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14466	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14467	AssertRCReturn(rc, rc);
14468
14469	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
14470	if (rcStrict == VINF_SUCCESS)
14471	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14472	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14473	{
14474	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14475	rcStrict = VINF_SUCCESS;
14476	}
14477
14478	return rcStrict;
14479	}
14480
14481
14482	/**
14483	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14484	*/
14485	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14486	{
14487	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14488
14489	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14490	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14491	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14492	AssertRCReturn(rc, rc);
14493
14494	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
14495	if (RT_SUCCESS(rcStrict))
14496	{
14497	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14498	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14499	rcStrict = VINF_SUCCESS;
14500	}
14501
14502	return rcStrict;
14503	}
14504
14505
14506	/**
14507	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14508	* VM-exit.
14509	*/
14510	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14511	{
14512	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14513	return VINF_EM_RESET;
14514	}
14515
14516
14517	/**
14518	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14519	*/
14520	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14521	{
14522	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14523
14524	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14525	AssertRCReturn(rc, rc);
14526
14527	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14528	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14529	rc = VINF_SUCCESS;
14530	else
14531	rc = VINF_EM_HALT;
14532
14533	if (rc != VINF_SUCCESS)
14534	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14535	return rc;
14536	}
14537
14538
14539	/**
14540	* VM-exit handler for instructions that result in a \#UD exception delivered to
14541	* the guest.
14542	*/
14543	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14544	{
14545	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14546	hmR0VmxSetPendingXcptUD(pVCpu);
14547	return VINF_SUCCESS;
14548	}
14549
14550
14551	/**
14552	* VM-exit handler for expiry of the VMX-preemption timer.
14553	*/
14554	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14555	{
14556	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14557
14558	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14559	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14560
14561	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14562	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14563	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14564	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14565	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14566	}
14567
14568
14569	/**
14570	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14571	*/
14572	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14573	{
14574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14575
14576	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14577	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14578	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14579	AssertRCReturn(rc, rc);
14580
14581	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
14582	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14583	: HM_CHANGED_RAISED_XCPT_MASK);
14584
14585	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14586	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14587
14588	return rcStrict;
14589	}
14590
14591
14592	/**
14593	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14594	*/
14595	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14596	{
14597	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14598
14599	#if 1
14600	/** @todo Use VM-exit instruction information. */
14601	return VERR_EM_INTERPRETER;
14602	#else
14603	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14604	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14605	hmR0VmxReadExitQualVmcs(pVmxTransient);
14606	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
14607	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14608	AssertRCReturn(rc, rc);
14609
14610	/* Paranoia. Ensure this has a memory operand. */
14611	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
14612
14613	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
14614	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14615	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
14616	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
14617
14618	RTGCPTR GCPtrDesc;
14619	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
14620
14621	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
14622	GCPtrDesc, uType);
14623	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14624	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14625	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14626	{
14627	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14628	rcStrict = VINF_SUCCESS;
14629	}
14630	return rcStrict;
14631	#endif
14632	}
14633
14634
14635	/**
14636	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14637	* VM-exit.
14638	*/
14639	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14640	{
14641	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14642	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14643	AssertRCReturn(rc, rc);
14644
14645	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14646	if (RT_FAILURE(rc))
14647	return rc;
14648
14649	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14650	NOREF(uInvalidReason);
14651
14652	#ifdef VBOX_STRICT
14653	uint32_t fIntrState;
14654	uint64_t u64Val;
14655	hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14656	hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14657	hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14658
14659	Log4(("uInvalidReason %u\n", uInvalidReason));
14660	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14661	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14662	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14663
14664	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
14665	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14666	rc = VMXReadVmcsNw(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
14667	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
14668	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
14669	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
14670	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
14671	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14672	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
14673	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
14674	rc = VMXReadVmcsNw(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
14675	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
14676	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14677	{
14678	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14679	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14680	}
14681	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
14682	#endif
14683
14684	return VERR_VMX_INVALID_GUEST_STATE;
14685	}
14686
14687	/**
14688	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14689	*/
14690	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14691	{
14692	/*
14693	* Cumulative notes of all recognized but unexpected VM-exits.
14694	*
14695	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
14696	* nested-paging is used.
14697	*
14698	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14699	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14700	* this function (and thereby stop VM execution) for handling such instructions.
14701	*
14702	*
14703	* VMX_EXIT_INIT_SIGNAL:
14704	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14705	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14706	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14707	*
14708	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14709	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14710	* See Intel spec. "23.8 Restrictions on VMX operation".
14711	*
14712	* VMX_EXIT_SIPI:
14713	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14714	* activity state is used. We don't make use of it as our guests don't have direct
14715	* access to the host local APIC.
14716	*
14717	* See Intel spec. 25.3 "Other Causes of VM-exits".
14718	*
14719	* VMX_EXIT_IO_SMI:
14720	* VMX_EXIT_SMI:
14721	* This can only happen if we support dual-monitor treatment of SMI, which can be
14722	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14723	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14724	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14725	*
14726	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14727	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14728	*
14729	* VMX_EXIT_ERR_MSR_LOAD:
14730	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14731	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14732	* execution.
14733	*
14734	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14735	*
14736	* VMX_EXIT_ERR_MACHINE_CHECK:
14737	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14738	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14739	* #MC exception abort class exception is raised. We thus cannot assume a
14740	* reasonable chance of continuing any sort of execution and we bail.
14741	*
14742	* See Intel spec. 15.1 "Machine-check Architecture".
14743	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14744	*
14745	* VMX_EXIT_PML_FULL:
14746	* VMX_EXIT_VIRTUALIZED_EOI:
14747	* VMX_EXIT_APIC_WRITE:
14748	* We do not currently support any of these features and thus they are all unexpected
14749	* VM-exits.
14750	*
14751	* VMX_EXIT_GDTR_IDTR_ACCESS:
14752	* VMX_EXIT_LDTR_TR_ACCESS:
14753	* VMX_EXIT_RDRAND:
14754	* VMX_EXIT_RSM:
14755	* VMX_EXIT_VMFUNC:
14756	* VMX_EXIT_ENCLS:
14757	* VMX_EXIT_RDSEED:
14758	* VMX_EXIT_XSAVES:
14759	* VMX_EXIT_XRSTORS:
14760	* VMX_EXIT_UMWAIT:
14761	* VMX_EXIT_TPAUSE:
14762	* These VM-exits are -not- caused unconditionally by execution of the corresponding
14763	* instruction. Any VM-exit for these instructions indicate a hardware problem,
14764	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14765	*
14766	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14767	*/
14768	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14769	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14770	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14771	}
14772
14773
14774	/**
14775	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14776	*/
14777	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14778	{
14779	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14780
14781	/** @todo Optimize this: We currently drag in the whole MSR state
14782	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14783	* MSRs required. That would require changes to IEM and possibly CPUM too.
14784	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14785	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14786	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14787	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14788	switch (idMsr)
14789	{
14790	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14791	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14792	}
14793
14794	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14795	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14796	AssertRCReturn(rc, rc);
14797
14798	Log4Func(("ecx=%#RX32\n", idMsr));
14799
14800	#ifdef VBOX_STRICT
14801	Assert(!pVmxTransient->fIsNestedGuest);
14802	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
14803	{
14804	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14805	&& idMsr != MSR_K6_EFER)
14806	{
14807	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14808	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14809	}
14810	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14811	{
14812	Assert(pVmcsInfo->pvMsrBitmap);
14813	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14814	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14815	{
14816	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14817	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14818	}
14819	}
14820	}
14821	#endif
14822
14823	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
14824	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14825	if (rcStrict == VINF_SUCCESS)
14826	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14827	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14828	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14829	{
14830	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14831	rcStrict = VINF_SUCCESS;
14832	}
14833	else
14834	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14835
14836	return rcStrict;
14837	}
14838
14839
14840	/**
14841	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14842	*/
14843	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14844	{
14845	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14846
14847	/** @todo Optimize this: We currently drag in the whole MSR state
14848	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14849	* MSRs required. That would require changes to IEM and possibly CPUM too.
14850	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14851	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14852	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14853
14854	/*
14855	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14856	* Although we don't need to fetch the base as it will be overwritten shortly, while
14857	* loading guest-state we would also load the entire segment register including limit
14858	* and attributes and thus we need to load them here.
14859	*/
14860	switch (idMsr)
14861	{
14862	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14863	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14864	}
14865
14866	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14867	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14868	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14869	AssertRCReturn(rc, rc);
14870
14871	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14872
14873	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
14874	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14875
14876	if (rcStrict == VINF_SUCCESS)
14877	{
14878	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14879
14880	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14881	if ( idMsr == MSR_IA32_APICBASE
14882	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14883	&& idMsr <= MSR_IA32_X2APIC_END))
14884	{
14885	/*
14886	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14887	* When full APIC register virtualization is implemented we'll have to make
14888	* sure APIC state is saved from the VMCS before IEM changes it.
14889	*/
14890	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14891	}
14892	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14893	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14894	else if (idMsr == MSR_K6_EFER)
14895	{
14896	/*
14897	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14898	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14899	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14900	*/
14901	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14902	}
14903
14904	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
14905	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14906	{
14907	switch (idMsr)
14908	{
14909	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14910	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14911	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14912	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14913	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14914	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14915	default:
14916	{
14917	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14918	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14919	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14920	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14921	break;
14922	}
14923	}
14924	}
14925	#ifdef VBOX_STRICT
14926	else
14927	{
14928	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14929	switch (idMsr)
14930	{
14931	case MSR_IA32_SYSENTER_CS:
14932	case MSR_IA32_SYSENTER_EIP:
14933	case MSR_IA32_SYSENTER_ESP:
14934	case MSR_K8_FS_BASE:
14935	case MSR_K8_GS_BASE:
14936	{
14937	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14938	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14939	}
14940
14941	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14942	default:
14943	{
14944	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14945	{
14946	/* EFER MSR writes are always intercepted. */
14947	if (idMsr != MSR_K6_EFER)
14948	{
14949	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14950	idMsr));
14951	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14952	}
14953	}
14954
14955	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14956	{
14957	Assert(pVmcsInfo->pvMsrBitmap);
14958	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14959	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14960	{
14961	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14962	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14963	}
14964	}
14965	break;
14966	}
14967	}
14968	}
14969	#endif /* VBOX_STRICT */
14970	}
14971	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14972	{
14973	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14974	rcStrict = VINF_SUCCESS;
14975	}
14976	else
14977	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14978
14979	return rcStrict;
14980	}
14981
14982
14983	/**
14984	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14985	*/
14986	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
14987	{
14988	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14989
14990	/** @todo The guest has likely hit a contended spinlock. We might want to
14991	* poke a schedule different guest VCPU. */
14992	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14993	if (RT_SUCCESS(rc))
14994	return VINF_EM_RAW_INTERRUPT;
14995
14996	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14997	return rc;
14998	}
14999
15000
15001	/**
15002	* VM-exit handler for when the TPR value is lowered below the specified
15003	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15004	*/
15005	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15006	{
15007	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15008	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15009
15010	/*
15011	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15012	* We'll re-evaluate pending interrupts and inject them before the next VM
15013	* entry so we can just continue execution here.
15014	*/
15015	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15016	return VINF_SUCCESS;
15017	}
15018
15019
15020	/**
15021	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15022	* VM-exit.
15023	*
15024	* @retval VINF_SUCCESS when guest execution can continue.
15025	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15026	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15027	* incompatible guest state for VMX execution (real-on-v86 case).
15028	*/
15029	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15030	{
15031	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15032	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15033
15034	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15035	hmR0VmxReadExitQualVmcs(pVmxTransient);
15036	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15037
15038	VBOXSTRICTRC rcStrict;
15039	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15040	uint64_t const uExitQual = pVmxTransient->uExitQual;
15041	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15042	switch (uAccessType)
15043	{
15044	/*
15045	* MOV to CRx.
15046	*/
15047	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15048	{
15049	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15050	AssertRCReturn(rc, rc);
15051
15052	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15053	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15054	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15055	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15056
15057	/*
15058	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15059	* - When nested paging isn't used.
15060	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15061	* - We are executing in the VM debug loop.
15062	*/
15063	Assert( iCrReg != 3
15064	\|\| !pVM->hm.s.fNestedPaging
15065	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15066	\|\| pVCpu->hm.s.fUsingDebugLoop);
15067
15068	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15069	Assert( iCrReg != 8
15070	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15071
15072	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15073	AssertMsg( rcStrict == VINF_SUCCESS
15074	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15075
15076	/*
15077	* This is a kludge for handling switches back to real mode when we try to use
15078	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15079	* deal with special selector values, so we have to return to ring-3 and run
15080	* there till the selector values are V86 mode compatible.
15081	*
15082	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15083	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15084	* this function.
15085	*/
15086	if ( iCrReg == 0
15087	&& rcStrict == VINF_SUCCESS
15088	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15089	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15090	&& (uOldCr0 & X86_CR0_PE)
15091	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15092	{
15093	/** @todo Check selectors rather than returning all the time. */
15094	Assert(!pVmxTransient->fIsNestedGuest);
15095	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15096	rcStrict = VINF_EM_RESCHEDULE_REM;
15097	}
15098	break;
15099	}
15100
15101	/*
15102	* MOV from CRx.
15103	*/
15104	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15105	{
15106	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15107	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15108
15109	/*
15110	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15111	* - When nested paging isn't used.
15112	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15113	* - We are executing in the VM debug loop.
15114	*/
15115	Assert( iCrReg != 3
15116	\|\| !pVM->hm.s.fNestedPaging
15117	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15118	\|\| pVCpu->hm.s.fUsingDebugLoop);
15119
15120	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15121	Assert( iCrReg != 8
15122	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15123
15124	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
15125	break;
15126	}
15127
15128	/*
15129	* CLTS (Clear Task-Switch Flag in CR0).
15130	*/
15131	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15132	{
15133	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
15134	break;
15135	}
15136
15137	/*
15138	* LMSW (Load Machine-Status Word into CR0).
15139	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15140	*/
15141	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15142	{
15143	RTGCPTR GCPtrEffDst;
15144	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
15145	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15146	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15147	if (fMemOperand)
15148	{
15149	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
15150	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15151	}
15152	else
15153	GCPtrEffDst = NIL_RTGCPTR;
15154	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15155	break;
15156	}
15157
15158	default:
15159	{
15160	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15161	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15162	}
15163	}
15164
15165	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15166	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15167	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15168
15169	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15170	NOREF(pVM);
15171	return rcStrict;
15172	}
15173
15174
15175	/**
15176	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15177	* VM-exit.
15178	*/
15179	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15180	{
15181	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15182	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15183
15184	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15185	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15186	hmR0VmxReadExitQualVmcs(pVmxTransient);
15187	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15188	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15189	\| CPUMCTX_EXTRN_EFER);
15190	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15191	AssertRCReturn(rc, rc);
15192
15193	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15194	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15195	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15196	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15197	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15198	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15199	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15200	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15201
15202	/*
15203	* Update exit history to see if this exit can be optimized.
15204	*/
15205	VBOXSTRICTRC rcStrict;
15206	PCEMEXITREC pExitRec = NULL;
15207	if ( !fGstStepping
15208	&& !fDbgStepping)
15209	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15210	!fIOString
15211	? !fIOWrite
15212	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15213	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15214	: !fIOWrite
15215	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15216	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15217	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15218	if (!pExitRec)
15219	{
15220	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15221	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15222
15223	uint32_t const cbValue = s_aIOSizes[uIOSize];
15224	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
15225	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15226	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15227	if (fIOString)
15228	{
15229	/*
15230	* INS/OUTS - I/O String instruction.
15231	*
15232	* Use instruction-information if available, otherwise fall back on
15233	* interpreting the instruction.
15234	*/
15235	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15236	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15237	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15238	if (fInsOutsInfo)
15239	{
15240	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15241	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15242	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15243	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15244	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15245	if (fIOWrite)
15246	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15247	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15248	else
15249	{
15250	/*
15251	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15252	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15253	* See Intel Instruction spec. for "INS".
15254	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15255	*/
15256	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15257	}
15258	}
15259	else
15260	rcStrict = IEMExecOne(pVCpu);
15261
15262	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15263	fUpdateRipAlready = true;
15264	}
15265	else
15266	{
15267	/*
15268	* IN/OUT - I/O instruction.
15269	*/
15270	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15271	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15272	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15273	if (fIOWrite)
15274	{
15275	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15276	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15277	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15278	&& !pCtx->eflags.Bits.u1TF)
15279	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15280	}
15281	else
15282	{
15283	uint32_t u32Result = 0;
15284	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15285	if (IOM_SUCCESS(rcStrict))
15286	{
15287	/* Save result of I/O IN instr. in AL/AX/EAX. */
15288	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15289	}
15290	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15291	&& !pCtx->eflags.Bits.u1TF)
15292	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15293	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15294	}
15295	}
15296
15297	if (IOM_SUCCESS(rcStrict))
15298	{
15299	if (!fUpdateRipAlready)
15300	{
15301	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15302	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15303	}
15304
15305	/*
15306	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15307	* while booting Fedora 17 64-bit guest.
15308	*
15309	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15310	*/
15311	if (fIOString)
15312	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15313
15314	/*
15315	* If any I/O breakpoints are armed, we need to check if one triggered
15316	* and take appropriate action.
15317	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15318	*/
15319	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15320	AssertRCReturn(rc, rc);
15321
15322	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15323	* execution engines about whether hyper BPs and such are pending. */
15324	uint32_t const uDr7 = pCtx->dr[7];
15325	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15326	&& X86_DR7_ANY_RW_IO(uDr7)
15327	&& (pCtx->cr4 & X86_CR4_DE))
15328	\|\| DBGFBpIsHwIoArmed(pVM)))
15329	{
15330	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15331
15332	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15333	VMMRZCallRing3Disable(pVCpu);
15334	HM_DISABLE_PREEMPT(pVCpu);
15335
15336	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15337
15338	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15339	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15340	{
15341	/* Raise #DB. */
15342	if (fIsGuestDbgActive)
15343	ASMSetDR6(pCtx->dr[6]);
15344	if (pCtx->dr[7] != uDr7)
15345	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15346
15347	hmR0VmxSetPendingXcptDB(pVCpu);
15348	}
15349	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15350	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15351	else if ( rcStrict2 != VINF_SUCCESS
15352	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15353	rcStrict = rcStrict2;
15354	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15355
15356	HM_RESTORE_PREEMPT();
15357	VMMRZCallRing3Enable(pVCpu);
15358	}
15359	}
15360
15361	#ifdef VBOX_STRICT
15362	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15363	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15364	Assert(!fIOWrite);
15365	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15366	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15367	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15368	Assert(fIOWrite);
15369	else
15370	{
15371	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15372	* statuses, that the VMM device and some others may return. See
15373	* IOM_SUCCESS() for guidance. */
15374	AssertMsg( RT_FAILURE(rcStrict)
15375	\|\| rcStrict == VINF_SUCCESS
15376	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15377	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15378	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15379	\|\| rcStrict == VINF_EM_RAW_TO_R3
15380	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15381	# endif
15382	}
15383	#endif
15384	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15385	}
15386	else
15387	{
15388	/*
15389	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15390	*/
15391	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15392	AssertRCReturn(rc2, rc2);
15393	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15394	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15395	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15396	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15397	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15398	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15399
15400	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15401	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15402
15403	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15404	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15405	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15406	}
15407	return rcStrict;
15408	}
15409
15410
15411	/**
15412	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15413	* VM-exit.
15414	*/
15415	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15416	{
15417	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15418
15419	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15420	hmR0VmxReadExitQualVmcs(pVmxTransient);
15421	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15422	{
15423	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15424	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15425	{
15426	uint32_t uErrCode;
15427	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15428	{
15429	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15430	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15431	}
15432	else
15433	uErrCode = 0;
15434
15435	RTGCUINTPTR GCPtrFaultAddress;
15436	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15437	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15438	else
15439	GCPtrFaultAddress = 0;
15440
15441	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15442
15443	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15444	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
15445
15446	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15447	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15448	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15449	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15450	}
15451	}
15452
15453	/* Fall back to the interpreter to emulate the task-switch. */
15454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15455	return VERR_EM_INTERPRETER;
15456	}
15457
15458
15459	/**
15460	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15461	*/
15462	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15463	{
15464	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15465
15466	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15467	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15468	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15469	AssertRC(rc);
15470	return VINF_EM_DBG_STEPPED;
15471	}
15472
15473
15474	/**
15475	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15476	*/
15477	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15478	{
15479	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15480	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15481
15482	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15483	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15484	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15485	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15486	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15487
15488	/*
15489	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15490	*/
15491	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15492	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15493	{
15494	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15495	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15496	{
15497	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15498	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15499	}
15500	}
15501	else
15502	{
15503	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15504	return rcStrict;
15505	}
15506
15507	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15508	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15509	hmR0VmxReadExitQualVmcs(pVmxTransient);
15510	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15511	AssertRCReturn(rc, rc);
15512
15513	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15514	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15515	switch (uAccessType)
15516	{
15517	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15518	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15519	{
15520	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15521	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15522	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15523
15524	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15525	GCPhys &= PAGE_BASE_GC_MASK;
15526	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15527	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15528	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15529
15530	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15531	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15532	rcStrict = IOMMMIOPhysHandler(pVM, pVCpu,
15533	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15534	CPUMCTX2CORE(pCtx), GCPhys);
15535	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15536	if ( rcStrict == VINF_SUCCESS
15537	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15538	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15539	{
15540	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15541	\| HM_CHANGED_GUEST_APIC_TPR);
15542	rcStrict = VINF_SUCCESS;
15543	}
15544	break;
15545	}
15546
15547	default:
15548	{
15549	Log4Func(("uAccessType=%#x\n", uAccessType));
15550	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15551	break;
15552	}
15553	}
15554
15555	if (rcStrict != VINF_SUCCESS)
15556	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15557	return rcStrict;
15558	}
15559
15560
15561	/**
15562	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15563	* VM-exit.
15564	*/
15565	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15566	{
15567	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15568	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15569
15570	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15571	if (!pVmxTransient->fIsNestedGuest)
15572	{
15573	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15574	if (pVmxTransient->fWasGuestDebugStateActive)
15575	{
15576	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15577	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15578	}
15579
15580	if ( !pVCpu->hm.s.fSingleInstruction
15581	&& !pVmxTransient->fWasHyperDebugStateActive)
15582	{
15583	Assert(!DBGFIsStepping(pVCpu));
15584	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15585
15586	/* Don't intercept MOV DRx any more. */
15587	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15588	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15589	AssertRC(rc);
15590
15591	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15592	VMMRZCallRing3Disable(pVCpu);
15593	HM_DISABLE_PREEMPT(pVCpu);
15594
15595	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15596	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15597	Assert(CPUMIsGuestDebugStateActive(pVCpu));
15598
15599	HM_RESTORE_PREEMPT();
15600	VMMRZCallRing3Enable(pVCpu);
15601
15602	#ifdef VBOX_WITH_STATISTICS
15603	hmR0VmxReadExitQualVmcs(pVmxTransient);
15604	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15605	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15606	else
15607	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15608	#endif
15609	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15610	return VINF_SUCCESS;
15611	}
15612	}
15613
15614	/*
15615	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15616	* The EFER MSR is always up-to-date.
15617	* Update the segment registers and DR7 from the CPU.
15618	*/
15619	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15620	hmR0VmxReadExitQualVmcs(pVmxTransient);
15621	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15622	AssertRCReturn(rc, rc);
15623	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15624
15625	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15626	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15627	{
15628	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15629	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15630	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15631	if (RT_SUCCESS(rc))
15632	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15633	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15634	}
15635	else
15636	{
15637	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15638	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15639	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15641	}
15642
15643	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15644	if (RT_SUCCESS(rc))
15645	{
15646	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15647	AssertRCReturn(rc2, rc2);
15648	return VINF_SUCCESS;
15649	}
15650	return rc;
15651	}
15652
15653
15654	/**
15655	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15656	* Conditional VM-exit.
15657	*/
15658	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15659	{
15660	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15661	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15662
15663	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15664	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15665	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15666	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15667	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15668
15669	/*
15670	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15671	*/
15672	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15673	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15674	{
15675	/*
15676	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15677	* instruction emulation to inject the original event. Otherwise, injecting the original event
15678	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
15679	*/
15680	if (!pVCpu->hm.s.Event.fPending)
15681	{ /* likely */ }
15682	else
15683	{
15684	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15685	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15686	/** @todo NSTVMX: Think about how this should be handled. */
15687	if (pVmxTransient->fIsNestedGuest)
15688	return VERR_VMX_IPE_3;
15689	#endif
15690	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15691	}
15692	}
15693	else
15694	{
15695	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15696	return rcStrict;
15697	}
15698
15699	/*
15700	* Get sufficient state and update the exit history entry.
15701	*/
15702	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15703	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15704	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15705	AssertRCReturn(rc, rc);
15706
15707	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15708	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15709	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15710	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15711	if (!pExitRec)
15712	{
15713	/*
15714	* If we succeed, resume guest execution.
15715	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15716	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15717	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15718	* weird case. See @bugref{6043}.
15719	*/
15720	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15721	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15722	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15723	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15724	if ( rcStrict == VINF_SUCCESS
15725	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15726	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15727	{
15728	/* Successfully handled MMIO operation. */
15729	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15730	\| HM_CHANGED_GUEST_APIC_TPR);
15731	rcStrict = VINF_SUCCESS;
15732	}
15733	}
15734	else
15735	{
15736	/*
15737	* Frequent exit or something needing probing. Call EMHistoryExec.
15738	*/
15739	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15740	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15741
15742	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15743	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15744
15745	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15746	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15747	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15748	}
15749	return rcStrict;
15750	}
15751
15752
15753	/**
15754	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15755	* VM-exit.
15756	*/
15757	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15758	{
15759	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15760	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15761
15762	hmR0VmxReadExitQualVmcs(pVmxTransient);
15763	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15764	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15765	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15766	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15767	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15768
15769	/*
15770	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15771	*/
15772	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15773	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15774	{
15775	/*
15776	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
15777	* we shall resolve the nested #PF and re-inject the original event.
15778	*/
15779	if (pVCpu->hm.s.Event.fPending)
15780	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
15781	}
15782	else
15783	{
15784	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15785	return rcStrict;
15786	}
15787
15788	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15789	hmR0VmxReadGuestPhysicalAddrVmcs(pVmxTransient);
15790	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15791	AssertRCReturn(rc, rc);
15792
15793	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15794	uint64_t const uExitQual = pVmxTransient->uExitQual;
15795	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
15796
15797	RTGCUINT uErrorCode = 0;
15798	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15799	uErrorCode \|= X86_TRAP_PF_ID;
15800	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15801	uErrorCode \|= X86_TRAP_PF_RW;
15802	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15803	uErrorCode \|= X86_TRAP_PF_P;
15804
15805	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
15806	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15807	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
15808
15809	/*
15810	* Handle the pagefault trap for the nested shadow table.
15811	*/
15812	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15813	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15814	TRPMResetTrap(pVCpu);
15815
15816	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15817	if ( rcStrict == VINF_SUCCESS
15818	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15819	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15820	{
15821	/* Successfully synced our nested page tables. */
15822	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15823	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15824	return VINF_SUCCESS;
15825	}
15826
15827	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15828	return rcStrict;
15829	}
15830
15831
15832	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15833	/**
15834	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15835	*/
15836	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15837	{
15838	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15839
15840	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15841	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15842	hmR0VmxReadExitQualVmcs(pVmxTransient);
15843	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15844	\| CPUMCTX_EXTRN_HWVIRT
15845	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15846	AssertRCReturn(rc, rc);
15847
15848	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15849
15850	VMXVEXITINFO ExitInfo;
15851	RT_ZERO(ExitInfo);
15852	ExitInfo.uReason = pVmxTransient->uExitReason;
15853	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15854	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15855	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15856	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15857
15858	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15859	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15860	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15861	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15862	{
15863	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15864	rcStrict = VINF_SUCCESS;
15865	}
15866	return rcStrict;
15867	}
15868
15869
15870	/**
15871	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15872	*/
15873	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15874	{
15875	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15876
15877	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15878	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15879	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15880	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15881	AssertRCReturn(rc, rc);
15882
15883	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15884
15885	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
15886	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
15887	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
15888	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15889	{
15890	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15891	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
15892	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15893	}
15894	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15895	return rcStrict;
15896	}
15897
15898
15899	/**
15900	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15901	*/
15902	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15903	{
15904	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15905
15906	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15907	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15908	hmR0VmxReadExitQualVmcs(pVmxTransient);
15909	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15910	\| CPUMCTX_EXTRN_HWVIRT
15911	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15912	AssertRCReturn(rc, rc);
15913
15914	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15915
15916	VMXVEXITINFO ExitInfo;
15917	RT_ZERO(ExitInfo);
15918	ExitInfo.uReason = pVmxTransient->uExitReason;
15919	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15920	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15921	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15922	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15923
15924	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15925	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15926	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15927	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15928	{
15929	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15930	rcStrict = VINF_SUCCESS;
15931	}
15932	return rcStrict;
15933	}
15934
15935
15936	/**
15937	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15938	*/
15939	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15940	{
15941	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15942
15943	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15944	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15945	hmR0VmxReadExitQualVmcs(pVmxTransient);
15946	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15947	\| CPUMCTX_EXTRN_HWVIRT
15948	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15949	AssertRCReturn(rc, rc);
15950
15951	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15952
15953	VMXVEXITINFO ExitInfo;
15954	RT_ZERO(ExitInfo);
15955	ExitInfo.uReason = pVmxTransient->uExitReason;
15956	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15957	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15958	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
15959	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15960
15961	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15962	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15963	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15964	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15965	{
15966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15967	rcStrict = VINF_SUCCESS;
15968	}
15969	return rcStrict;
15970	}
15971
15972
15973	/**
15974	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
15975	*/
15976	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
15977	{
15978	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15979
15980	/*
15981	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
15982	* thus might not need to import the shadow VMCS state, it's safer just in case
15983	* code elsewhere dares look at unsynced VMCS fields.
15984	*/
15985	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15986	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15987	hmR0VmxReadExitQualVmcs(pVmxTransient);
15988	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15989	\| CPUMCTX_EXTRN_HWVIRT
15990	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15991	AssertRCReturn(rc, rc);
15992
15993	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15994
15995	VMXVEXITINFO ExitInfo;
15996	RT_ZERO(ExitInfo);
15997	ExitInfo.uReason = pVmxTransient->uExitReason;
15998	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15999	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16000	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16001	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16002	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16003
16004	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16005	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16006	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16007	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16008	{
16009	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16010	rcStrict = VINF_SUCCESS;
16011	}
16012	return rcStrict;
16013	}
16014
16015
16016	/**
16017	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16018	*/
16019	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16020	{
16021	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16022
16023	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16024	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16025	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16026	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16027	AssertRCReturn(rc, rc);
16028
16029	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16030
16031	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16032	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
16033	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16034	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16035	{
16036	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16037	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16038	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16039	}
16040	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16041	return rcStrict;
16042	}
16043
16044
16045	/**
16046	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16047	*/
16048	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16049	{
16050	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16051
16052	/*
16053	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16054	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16055	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16056	*/
16057	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16058	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16059	hmR0VmxReadExitQualVmcs(pVmxTransient);
16060	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16061	\| CPUMCTX_EXTRN_HWVIRT
16062	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16063	AssertRCReturn(rc, rc);
16064
16065	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16066
16067	VMXVEXITINFO ExitInfo;
16068	RT_ZERO(ExitInfo);
16069	ExitInfo.uReason = pVmxTransient->uExitReason;
16070	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16071	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16072	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16073	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16074	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16075
16076	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16077	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16078	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16079	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16080	{
16081	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16082	rcStrict = VINF_SUCCESS;
16083	}
16084	return rcStrict;
16085	}
16086
16087
16088	/**
16089	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16090	*/
16091	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16092	{
16093	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16094
16095	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16096	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16097	\| CPUMCTX_EXTRN_HWVIRT
16098	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16099	AssertRCReturn(rc, rc);
16100
16101	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16102
16103	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
16104	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16105	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16106	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16107	{
16108	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16109	rcStrict = VINF_SUCCESS;
16110	}
16111	return rcStrict;
16112	}
16113
16114
16115	/**
16116	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16117	*/
16118	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16119	{
16120	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16121
16122	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16123	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16124	hmR0VmxReadExitQualVmcs(pVmxTransient);
16125	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16126	\| CPUMCTX_EXTRN_HWVIRT
16127	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16128	AssertRCReturn(rc, rc);
16129
16130	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16131
16132	VMXVEXITINFO ExitInfo;
16133	RT_ZERO(ExitInfo);
16134	ExitInfo.uReason = pVmxTransient->uExitReason;
16135	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16136	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16137	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16138	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16139
16140	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16141	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16142	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16143	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16144	{
16145	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16146	rcStrict = VINF_SUCCESS;
16147	}
16148	return rcStrict;
16149	}
16150
16151
16152	/**
16153	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16154	*/
16155	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16156	{
16157	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16158
16159	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16160	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16161	hmR0VmxReadExitQualVmcs(pVmxTransient);
16162	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16163	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16164	AssertRCReturn(rc, rc);
16165
16166	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16167
16168	VMXVEXITINFO ExitInfo;
16169	RT_ZERO(ExitInfo);
16170	ExitInfo.uReason = pVmxTransient->uExitReason;
16171	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16172	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16173	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16174	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16175
16176	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16177	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16178	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16179	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16180	{
16181	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16182	rcStrict = VINF_SUCCESS;
16183	}
16184	return rcStrict;
16185	}
16186	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16187	/** @} */
16188
16189
16190	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16191	/** @name Nested-guest VM-exit handlers.
16192	* @{
16193	*/
16194	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16195	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16196	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16197
16198	/**
16199	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16200	* Conditional VM-exit.
16201	*/
16202	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16203	{
16204	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16205
16206	hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16207
16208	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16209	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16210	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16211
16212	switch (uExitIntType)
16213	{
16214	/*
16215	* Physical NMIs:
16216	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16217	*/
16218	case VMX_EXIT_INT_INFO_TYPE_NMI:
16219	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16220
16221	/*
16222	* Hardware exceptions,
16223	* Software exceptions,
16224	* Privileged software exceptions:
16225	* Figure out if the exception must be delivered to the guest or the nested-guest.
16226	*/
16227	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16228	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16229	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16230	{
16231	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16232	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16233	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16234	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16235
16236	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16237	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16238	pVmxTransient->uExitIntErrorCode);
16239	if (fIntercept)
16240	{
16241	/* Exit qualification is required for debug and page-fault exceptions. */
16242	hmR0VmxReadExitQualVmcs(pVmxTransient);
16243
16244	/*
16245	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16246	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16247	* length. However, if delivery of a software interrupt, software exception or privileged
16248	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16249	*/
16250	VMXVEXITINFO ExitInfo;
16251	RT_ZERO(ExitInfo);
16252	ExitInfo.uReason = pVmxTransient->uExitReason;
16253	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16254	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16255
16256	VMXVEXITEVENTINFO ExitEventInfo;
16257	RT_ZERO(ExitEventInfo);
16258	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16259	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16260	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16261	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16262
16263	#ifdef DEBUG_ramshankar
16264	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16265	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
16266	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16267	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16268	{
16269	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
16270	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
16271	}
16272	#endif
16273	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16274	}
16275
16276	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16277	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16278	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16279	}
16280
16281	/*
16282	* Software interrupts:
16283	* VM-exits cannot be caused by software interrupts.
16284	*
16285	* External interrupts:
16286	* This should only happen when "acknowledge external interrupts on VM-exit"
16287	* control is set. However, we never set this when executing a guest or
16288	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16289	* the guest.
16290	*/
16291	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16292	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16293	default:
16294	{
16295	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16296	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16297	}
16298	}
16299	}
16300
16301
16302	/**
16303	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16304	* Unconditional VM-exit.
16305	*/
16306	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16307	{
16308	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16309	return IEMExecVmxVmexitTripleFault(pVCpu);
16310	}
16311
16312
16313	/**
16314	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16315	*/
16316	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16317	{
16318	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16319
16320	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16321	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16322	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16323	}
16324
16325
16326	/**
16327	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16328	*/
16329	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16330	{
16331	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16332
16333	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16334	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16335	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16336	}
16337
16338
16339	/**
16340	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16341	* Unconditional VM-exit.
16342	*/
16343	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16344	{
16345	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16346
16347	hmR0VmxReadExitQualVmcs(pVmxTransient);
16348	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16349	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16350	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16351
16352	VMXVEXITINFO ExitInfo;
16353	RT_ZERO(ExitInfo);
16354	ExitInfo.uReason = pVmxTransient->uExitReason;
16355	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16356	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16357
16358	VMXVEXITEVENTINFO ExitEventInfo;
16359	RT_ZERO(ExitEventInfo);
16360	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16361	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16362	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16363	}
16364
16365
16366	/**
16367	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16368	*/
16369	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16370	{
16371	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16372
16373	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16374	{
16375	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16376	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16377	}
16378	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16379	}
16380
16381
16382	/**
16383	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16384	*/
16385	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16386	{
16387	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16388
16389	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16390	{
16391	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16392	hmR0VmxReadExitQualVmcs(pVmxTransient);
16393
16394	VMXVEXITINFO ExitInfo;
16395	RT_ZERO(ExitInfo);
16396	ExitInfo.uReason = pVmxTransient->uExitReason;
16397	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16398	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16399	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16400	}
16401	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16402	}
16403
16404
16405	/**
16406	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16407	*/
16408	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16409	{
16410	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16411
16412	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16413	{
16414	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16415	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16416	}
16417	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16418	}
16419
16420
16421	/**
16422	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16423	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16424	*/
16425	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16426	{
16427	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16428
16429	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16430	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16431
16432	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16433
16434	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16435	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16436	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16437
16438	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16439	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16440	u64VmcsField &= UINT64_C(0xffffffff);
16441
16442	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16443	{
16444	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16445	hmR0VmxReadExitQualVmcs(pVmxTransient);
16446
16447	VMXVEXITINFO ExitInfo;
16448	RT_ZERO(ExitInfo);
16449	ExitInfo.uReason = pVmxTransient->uExitReason;
16450	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16451	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16452	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16453	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16454	}
16455
16456	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16457	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16458	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16459	}
16460
16461
16462	/**
16463	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16464	*/
16465	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16466	{
16467	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16468
16469	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16470	{
16471	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16472	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16473	}
16474
16475	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16476	}
16477
16478
16479	/**
16480	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16481	* Conditional VM-exit.
16482	*/
16483	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16484	{
16485	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16486
16487	hmR0VmxReadExitQualVmcs(pVmxTransient);
16488	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16489
16490	VBOXSTRICTRC rcStrict;
16491	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16492	switch (uAccessType)
16493	{
16494	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16495	{
16496	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16497	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16498	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16499	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16500
16501	bool fIntercept;
16502	switch (iCrReg)
16503	{
16504	case 0:
16505	case 4:
16506	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16507	break;
16508
16509	case 3:
16510	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16511	break;
16512
16513	case 8:
16514	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16515	break;
16516
16517	default:
16518	fIntercept = false;
16519	break;
16520	}
16521	if (fIntercept)
16522	{
16523	VMXVEXITINFO ExitInfo;
16524	RT_ZERO(ExitInfo);
16525	ExitInfo.uReason = pVmxTransient->uExitReason;
16526	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16527	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16528	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16529	}
16530	else
16531	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16532	break;
16533	}
16534
16535	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16536	{
16537	/*
16538	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16539	* CR2 reads do not cause a VM-exit.
16540	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16541	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16542	*/
16543	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16544	if ( iCrReg == 3
16545	\|\| iCrReg == 8)
16546	{
16547	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16548	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16549	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16550	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16551	{
16552	VMXVEXITINFO ExitInfo;
16553	RT_ZERO(ExitInfo);
16554	ExitInfo.uReason = pVmxTransient->uExitReason;
16555	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16556	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16557	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16558	}
16559	else
16560	{
16561	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16562	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
16563	}
16564	}
16565	else
16566	{
16567	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16568	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16569	}
16570	break;
16571	}
16572
16573	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16574	{
16575	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16576	Assert(pVmcsNstGst);
16577	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16578	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16579	if ( (uGstHostMask & X86_CR0_TS)
16580	&& (uReadShadow & X86_CR0_TS))
16581	{
16582	VMXVEXITINFO ExitInfo;
16583	RT_ZERO(ExitInfo);
16584	ExitInfo.uReason = pVmxTransient->uExitReason;
16585	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16586	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16587	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16588	}
16589	else
16590	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
16591	break;
16592	}
16593
16594	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16595	{
16596	RTGCPTR GCPtrEffDst;
16597	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16598	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16599	if (fMemOperand)
16600	{
16601	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16602	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16603	}
16604	else
16605	GCPtrEffDst = NIL_RTGCPTR;
16606
16607	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16608	{
16609	VMXVEXITINFO ExitInfo;
16610	RT_ZERO(ExitInfo);
16611	ExitInfo.uReason = pVmxTransient->uExitReason;
16612	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16613	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16614	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16615	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16616	}
16617	else
16618	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
16619	break;
16620	}
16621
16622	default:
16623	{
16624	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16625	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16626	}
16627	}
16628
16629	if (rcStrict == VINF_IEM_RAISED_XCPT)
16630	{
16631	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16632	rcStrict = VINF_SUCCESS;
16633	}
16634	return rcStrict;
16635	}
16636
16637
16638	/**
16639	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16640	* Conditional VM-exit.
16641	*/
16642	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16643	{
16644	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16645
16646	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16647	{
16648	hmR0VmxReadExitQualVmcs(pVmxTransient);
16649	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16650
16651	VMXVEXITINFO ExitInfo;
16652	RT_ZERO(ExitInfo);
16653	ExitInfo.uReason = pVmxTransient->uExitReason;
16654	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16655	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16656	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16657	}
16658	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16659	}
16660
16661
16662	/**
16663	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16664	* Conditional VM-exit.
16665	*/
16666	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16667	{
16668	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16669
16670	hmR0VmxReadExitQualVmcs(pVmxTransient);
16671
16672	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16673	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16674	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16675
16676	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16677	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16678	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16679	{
16680	/*
16681	* IN/OUT instruction:
16682	* - Provides VM-exit instruction length.
16683	*
16684	* INS/OUTS instruction:
16685	* - Provides VM-exit instruction length.
16686	* - Provides Guest-linear address.
16687	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16688	*/
16689	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
16690	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16691
16692	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16693	pVmxTransient->ExitInstrInfo.u = 0;
16694	pVmxTransient->uGuestLinearAddr = 0;
16695
16696	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16697	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16698	if (fIOString)
16699	{
16700	hmR0VmxReadGuestLinearAddrVmcs(pVmxTransient);
16701	if (fVmxInsOutsInfo)
16702	{
16703	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16704	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16705	}
16706	}
16707
16708	VMXVEXITINFO ExitInfo;
16709	RT_ZERO(ExitInfo);
16710	ExitInfo.uReason = pVmxTransient->uExitReason;
16711	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16712	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16713	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16714	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16715	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16716	}
16717	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16718	}
16719
16720
16721	/**
16722	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16723	*/
16724	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16725	{
16726	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16727
16728	uint32_t fMsrpm;
16729	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16730	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16731	else
16732	fMsrpm = VMXMSRPM_EXIT_RD;
16733
16734	if (fMsrpm & VMXMSRPM_EXIT_RD)
16735	{
16736	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16737	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16738	}
16739	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16740	}
16741
16742
16743	/**
16744	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16745	*/
16746	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16747	{
16748	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16749
16750	uint32_t fMsrpm;
16751	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16752	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16753	else
16754	fMsrpm = VMXMSRPM_EXIT_WR;
16755
16756	if (fMsrpm & VMXMSRPM_EXIT_WR)
16757	{
16758	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16759	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16760	}
16761	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16762	}
16763
16764
16765	/**
16766	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16767	*/
16768	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16769	{
16770	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16771
16772	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16773	{
16774	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16775	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16776	}
16777	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16778	}
16779
16780
16781	/**
16782	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16783	* VM-exit.
16784	*/
16785	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16786	{
16787	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16788
16789	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16790	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16791	VMXVEXITINFO ExitInfo;
16792	RT_ZERO(ExitInfo);
16793	ExitInfo.uReason = pVmxTransient->uExitReason;
16794	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16795	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16796	}
16797
16798
16799	/**
16800	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16801	*/
16802	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16803	{
16804	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16805
16806	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16807	{
16808	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16809	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16810	}
16811	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16812	}
16813
16814
16815	/**
16816	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16817	*/
16818	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16819	{
16820	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16821
16822	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16823	* PAUSE when executing a nested-guest? If it does not, we would not need
16824	* to check for the intercepts here. Just call VM-exit... */
16825
16826	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16827	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16828	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16829	{
16830	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16831	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16832	}
16833	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16834	}
16835
16836
16837	/**
16838	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16839	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16840	*/
16841	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16842	{
16843	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16844
16845	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16846	{
16847	hmR0VmxReadGuestPendingDbgXctps(pVmxTransient);
16848	VMXVEXITINFO ExitInfo;
16849	RT_ZERO(ExitInfo);
16850	ExitInfo.uReason = pVmxTransient->uExitReason;
16851	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
16852	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
16853	}
16854	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16855	}
16856
16857
16858	/**
16859	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16860	* VM-exit.
16861	*/
16862	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16863	{
16864	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16865
16866	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16867	hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16868	hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16869	hmR0VmxReadExitQualVmcs(pVmxTransient);
16870
16871	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16872
16873	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
16874	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
16875
16876	VMXVEXITINFO ExitInfo;
16877	RT_ZERO(ExitInfo);
16878	ExitInfo.uReason = pVmxTransient->uExitReason;
16879	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16880	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16881
16882	VMXVEXITEVENTINFO ExitEventInfo;
16883	RT_ZERO(ExitEventInfo);
16884	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16885	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16886	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
16887	}
16888
16889
16890	/**
16891	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
16892	* Conditional VM-exit.
16893	*/
16894	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16895	{
16896	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16897
16898	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
16899	hmR0VmxReadExitQualVmcs(pVmxTransient);
16900	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16901	}
16902
16903
16904	/**
16905	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
16906	* Conditional VM-exit.
16907	*/
16908	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16909	{
16910	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16911
16912	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
16913	hmR0VmxReadExitQualVmcs(pVmxTransient);
16914	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16915	}
16916
16917
16918	/**
16919	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16920	*/
16921	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16922	{
16923	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16924
16925	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16926	{
16927	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16928	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16929	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16930	}
16931	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16932	}
16933
16934
16935	/**
16936	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16937	*/
16938	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16939	{
16940	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16941
16942	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16943	{
16944	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16945	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
16946	}
16947	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16948	}
16949
16950
16951	/**
16952	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16953	*/
16954	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16955	{
16956	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16957
16958	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16959	{
16960	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
16961	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16962	hmR0VmxReadExitQualVmcs(pVmxTransient);
16963	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16964
16965	VMXVEXITINFO ExitInfo;
16966	RT_ZERO(ExitInfo);
16967	ExitInfo.uReason = pVmxTransient->uExitReason;
16968	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
16969	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16970	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16971	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16972	}
16973	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
16974	}
16975
16976
16977	/**
16978	* Nested-guest VM-exit handler for invalid-guest state
16979	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
16980	*/
16981	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
16982	{
16983	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16984
16985	/*
16986	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
16987	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
16988	* Handle it like it's in an invalid guest state of the outer guest.
16989	*
16990	* When the fast path is implemented, this should be changed to cause the corresponding
16991	* nested-guest VM-exit.
16992	*/
16993	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
16994	}
16995
16996
16997	/**
16998	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
16999	* and only provide the instruction length.
17000	*
17001	* Unconditional VM-exit.
17002	*/
17003	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17004	{
17005	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17006
17007	#ifdef VBOX_STRICT
17008	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17009	switch (pVmxTransient->uExitReason)
17010	{
17011	case VMX_EXIT_ENCLS:
17012	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17013	break;
17014
17015	case VMX_EXIT_VMFUNC:
17016	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17017	break;
17018	}
17019	#endif
17020
17021	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17022	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
17023	}
17024
17025
17026	/**
17027	* Nested-guest VM-exit handler for instructions that provide instruction length as
17028	* well as more information.
17029	*
17030	* Unconditional VM-exit.
17031	*/
17032	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
17033	{
17034	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17035
17036	#ifdef VBOX_STRICT
17037	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17038	switch (pVmxTransient->uExitReason)
17039	{
17040	case VMX_EXIT_GDTR_IDTR_ACCESS:
17041	case VMX_EXIT_LDTR_TR_ACCESS:
17042	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17043	break;
17044
17045	case VMX_EXIT_RDRAND:
17046	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17047	break;
17048
17049	case VMX_EXIT_RDSEED:
17050	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17051	break;
17052
17053	case VMX_EXIT_XSAVES:
17054	case VMX_EXIT_XRSTORS:
17055	/** @todo NSTVMX: Verify XSS-bitmap. */
17056	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17057	break;
17058
17059	case VMX_EXIT_UMWAIT:
17060	case VMX_EXIT_TPAUSE:
17061	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17062	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17063	break;
17064	}
17065	#endif
17066
17067	hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17068	hmR0VmxReadExitQualVmcs(pVmxTransient);
17069	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17070
17071	VMXVEXITINFO ExitInfo;
17072	RT_ZERO(ExitInfo);
17073	ExitInfo.uReason = pVmxTransient->uExitReason;
17074	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
17075	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17076	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17077	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17078	}
17079
17080	/** @} */
17081	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17082

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 80840

Download in other formats: