HMVMXR0.cpp@ 79660

Last change on this file since 79660 was 79660, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Grammar.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 730.5 KB

Line
1	/* $Id: HMVMXR0.cpp 79660 2019-07-10 08:39:52Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/gim.h>
38	#include <VBox/vmm/apic.h>
39	#ifdef VBOX_WITH_REM
40	# include <VBox/vmm/rem.h>
41	#endif
42	#include "HMInternal.h"
43	#include <VBox/vmm/vm.h>
44	#include <VBox/vmm/hmvmxinline.h>
45	#include "HMVMXR0.h"
46	#include "dtrace/VBoxVMM.h"
47
48	#ifdef DEBUG_ramshankar
49	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
50	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
51	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
52	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
53	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
54	# define HMVMX_ALWAYS_TRAP_PF
55	# define HMVMX_ALWAYS_FLUSH_TLB
56	# define HMVMX_ALWAYS_SWAP_EFER
57	#endif
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	/** Use the function table. */
64	#define HMVMX_USE_FUNCTION_TABLE
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name HMVMX_READ_XXX
73	* Flags to skip redundant reads of some common VMCS fields that are not part of
74	* the guest-CPU or VCPU state but are needed while handling VM-exits.
75	*/
76	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
77	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
78	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
79	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
80	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
81	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
82	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
83	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
84	/** @} */
85
86	/**
87	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
88	* guest using hardware-assisted VMX.
89	*
90	* This excludes state like GPRs (other than RSP) which are always are
91	* swapped and restored across the world-switch and also registers like EFER,
92	* MSR which cannot be modified by the guest without causing a VM-exit.
93	*/
94	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
95	\| CPUMCTX_EXTRN_RFLAGS \
96	\| CPUMCTX_EXTRN_RSP \
97	\| CPUMCTX_EXTRN_SREG_MASK \
98	\| CPUMCTX_EXTRN_TABLE_MASK \
99	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
100	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
101	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
102	\| CPUMCTX_EXTRN_TSC_AUX \
103	\| CPUMCTX_EXTRN_OTHER_MSRS \
104	\| CPUMCTX_EXTRN_CR0 \
105	\| CPUMCTX_EXTRN_CR3 \
106	\| CPUMCTX_EXTRN_CR4 \
107	\| CPUMCTX_EXTRN_DR7 \
108	\| CPUMCTX_EXTRN_HWVIRT \
109	\| CPUMCTX_EXTRN_HM_VMX_MASK)
110
111	/**
112	* Exception bitmap mask for real-mode guests (real-on-v86).
113	*
114	* We need to intercept all exceptions manually except:
115	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
116	* due to bugs in Intel CPUs.
117	* - \#PF need not be intercepted even in real-mode if we have nested paging
118	* support.
119	*/
120	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
121	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
122	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
123	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
124	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
125	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
126	\| RT_BIT(X86_XCPT_XF))
127
128	/** Maximum VM-instruction error number. */
129	#define HMVMX_INSTR_ERROR_MAX 28
130
131	/** Profiling macro. */
132	#ifdef HM_PROFILE_EXIT_DISPATCH
133	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
135	#else
136	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
137	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
138	#endif
139
140	/** Assert that preemption is disabled or covered by thread-context hooks. */
141	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
142	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
143
144	/** Assert that we haven't migrated CPUs when thread-context hooks are not
145	* used. */
146	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
147	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
148	("Illegal migration! Entered on CPU %u Current %u\n", \
149	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
150
151	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
152	* context. */
153	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
154	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
155	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
156
157	/** Log the VM-exit reason with an easily visible marker to identify it in a
158	* potential sea of logging data. */
159	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
160	do { \
161	Log4(("VM-exit: vcpu[%RU32] reason=%#x -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", \
162	(a_pVCpu)->idCpu, (a_uExitReason))); \
163	} while (0) \
164
165
166	/*********************************************************************************************************************************
167	* Structures and Typedefs *
168	*********************************************************************************************************************************/
169	/**
170	* VMX per-VCPU transient state.
171	*
172	* A state structure for holding miscellaneous information across
173	* VMX non-root operation and restored after the transition.
174	*/
175	typedef struct VMXTRANSIENT
176	{
177	/** The host's rflags/eflags. */
178	RTCCUINTREG fEFlags;
179	#if HC_ARCH_BITS == 32
180	uint32_t u32Alignment0;
181	#endif
182	/** The guest's TPR value used for TPR shadowing. */
183	uint8_t u8GuestTpr;
184	/** Alignment. */
185	uint8_t abAlignment0[7];
186
187	/** The basic VM-exit reason. */
188	uint16_t uExitReason;
189	/** Alignment. */
190	uint16_t u16Alignment0;
191	/** The VM-exit interruption error code. */
192	uint32_t uExitIntErrorCode;
193	/** The VM-exit exit code qualification. */
194	uint64_t uExitQual;
195	/** The Guest-linear address. */
196	uint64_t uGuestLinearAddr;
197
198	/** The VM-exit interruption-information field. */
199	uint32_t uExitIntInfo;
200	/** The VM-exit instruction-length field. */
201	uint32_t cbInstr;
202	/** The VM-exit instruction-information field. */
203	VMXEXITINSTRINFO ExitInstrInfo;
204	/** Whether the VM-entry failed or not. */
205	bool fVMEntryFailed;
206	/** Whether we are currently executing a nested-guest. */
207	bool fIsNestedGuest;
208	/** Alignment. */
209	uint8_t abAlignment1[2];
210
211	/** The VM-entry interruption-information field. */
212	uint32_t uEntryIntInfo;
213	/** The VM-entry exception error code field. */
214	uint32_t uEntryXcptErrorCode;
215	/** The VM-entry instruction length field. */
216	uint32_t cbEntryInstr;
217
218	/** IDT-vectoring information field. */
219	uint32_t uIdtVectoringInfo;
220	/** IDT-vectoring error code. */
221	uint32_t uIdtVectoringErrorCode;
222
223	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
224	uint32_t fVmcsFieldsRead;
225
226	/** Whether the guest debug state was active at the time of VM-exit. */
227	bool fWasGuestDebugStateActive;
228	/** Whether the hyper debug state was active at the time of VM-exit. */
229	bool fWasHyperDebugStateActive;
230	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
231	bool fUpdatedTscOffsettingAndPreemptTimer;
232	/** Whether the VM-exit was caused by a page-fault during delivery of a
233	* contributory exception or a page-fault. */
234	bool fVectoringDoublePF;
235	/** Whether the VM-exit was caused by a page-fault during delivery of an
236	* external interrupt or NMI. */
237	bool fVectoringPF;
238	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
239	* area after VM-exit. */
240	bool fRemoveTscAuxMsr;
241	bool afAlignment0[2];
242
243	/** The VMCS info. object. */
244	PVMXVMCSINFO pVmcsInfo;
245	} VMXTRANSIENT;
246	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
247	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
248	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
249	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
250	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
251	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
252	/** Pointer to VMX transient state. */
253	typedef VMXTRANSIENT *PVMXTRANSIENT;
254	/** Pointer to a const VMX transient state. */
255	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
256
257	/**
258	* Memory operand read or write access.
259	*/
260	typedef enum VMXMEMACCESS
261	{
262	VMXMEMACCESS_READ = 0,
263	VMXMEMACCESS_WRITE = 1
264	} VMXMEMACCESS;
265
266	/**
267	* VMX VM-exit handler.
268	*
269	* @returns Strict VBox status code (i.e. informational status codes too).
270	* @param pVCpu The cross context virtual CPU structure.
271	* @param pVmxTransient The VMX-transient structure.
272	*/
273	#ifndef HMVMX_USE_FUNCTION_TABLE
274	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
275	#else
276	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
277	/** Pointer to VM-exit handler. */
278	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
279	#endif
280
281	/**
282	* VMX VM-exit handler, non-strict status code.
283	*
284	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
285	*
286	* @returns VBox status code, no informational status code returned.
287	* @param pVCpu The cross context virtual CPU structure.
288	* @param pVmxTransient The VMX-transient structure.
289	*
290	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
291	* use of that status code will be replaced with VINF_EM_SOMETHING
292	* later when switching over to IEM.
293	*/
294	#ifndef HMVMX_USE_FUNCTION_TABLE
295	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
296	#else
297	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
298	#endif
299
300
301	/*********************************************************************************************************************************
302	* Internal Functions *
303	*********************************************************************************************************************************/
304	#ifndef HMVMX_USE_FUNCTION_TABLE
305	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
306	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
307	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
308	#else
309	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
310	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
311	#endif
312	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
313	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	#endif
315
316	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
317	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
318	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
319	#endif
320
321	/** @name VM-exit handler prototypes.
322	* @{
323	*/
324	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
325	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
326	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
327	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
328	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
329	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
330	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
331	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
332	static FNVMXEXITHANDLER hmR0VmxExitHlt;
333	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
334	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
335	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
336	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
337	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
338	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
339	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
340	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
341	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
342	static FNVMXEXITHANDLER hmR0VmxExitVmread;
343	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
344	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
345	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
346	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
347	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
348	#endif
349	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
350	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
351	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
352	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
353	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
354	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
355	static FNVMXEXITHANDLER hmR0VmxExitMwait;
356	static FNVMXEXITHANDLER hmR0VmxExitMtf;
357	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
358	static FNVMXEXITHANDLER hmR0VmxExitPause;
359	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
360	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
361	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
362	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
363	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
364	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
366	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
367	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
371	/** @} */
372
373	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
374	/** @name Nested-guest VM-exit handler prototypes.
375	* @{
376	*/
377	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
378	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
379	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
380	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
381	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
382	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
383	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
384	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
385	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
386	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
387	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
388	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
389	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
390	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
391	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
392	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
393	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
394	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
395	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
397	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
398	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
399	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
400	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
401	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
402	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
404	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
405	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
406	/** @} */
407	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
408
409
410	/*********************************************************************************************************************************
411	* Global Variables *
412	*********************************************************************************************************************************/
413	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
414	/**
415	* Array of all VMCS fields.
416	* Any fields added to the VT-x spec. should be added here.
417	*
418	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
419	* of nested-guests.
420	*/
421	static const uint32_t g_aVmcsFields[] =
422	{
423	/* 16-bit control fields. */
424	VMX_VMCS16_VPID,
425	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
426	VMX_VMCS16_EPTP_INDEX,
427
428	/* 16-bit guest-state fields. */
429	VMX_VMCS16_GUEST_ES_SEL,
430	VMX_VMCS16_GUEST_CS_SEL,
431	VMX_VMCS16_GUEST_SS_SEL,
432	VMX_VMCS16_GUEST_DS_SEL,
433	VMX_VMCS16_GUEST_FS_SEL,
434	VMX_VMCS16_GUEST_GS_SEL,
435	VMX_VMCS16_GUEST_LDTR_SEL,
436	VMX_VMCS16_GUEST_TR_SEL,
437	VMX_VMCS16_GUEST_INTR_STATUS,
438	VMX_VMCS16_GUEST_PML_INDEX,
439
440	/* 16-bits host-state fields. */
441	VMX_VMCS16_HOST_ES_SEL,
442	VMX_VMCS16_HOST_CS_SEL,
443	VMX_VMCS16_HOST_SS_SEL,
444	VMX_VMCS16_HOST_DS_SEL,
445	VMX_VMCS16_HOST_FS_SEL,
446	VMX_VMCS16_HOST_GS_SEL,
447	VMX_VMCS16_HOST_TR_SEL,
448
449	/* 64-bit control fields. */
450	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
451	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
452	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
453	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
454	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
455	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
456	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
457	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
458	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
459	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
460	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
461	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
462	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
463	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
464	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
465	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
466	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
467	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
468	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
469	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
470	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
471	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
472	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
473	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
474	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
475	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
476	VMX_VMCS64_CTRL_EPTP_FULL,
477	VMX_VMCS64_CTRL_EPTP_HIGH,
478	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
479	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
480	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
481	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
482	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
483	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
484	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
485	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
486	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
487	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
488	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
489	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
490	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
491	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
492	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
493	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
494	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
495	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
496	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
497	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
498	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
499	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
500
501	/* 64-bit read-only data fields. */
502	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
503	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
504
505	/* 64-bit guest-state fields. */
506	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
507	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
508	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
509	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
510	VMX_VMCS64_GUEST_PAT_FULL,
511	VMX_VMCS64_GUEST_PAT_HIGH,
512	VMX_VMCS64_GUEST_EFER_FULL,
513	VMX_VMCS64_GUEST_EFER_HIGH,
514	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
515	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
516	VMX_VMCS64_GUEST_PDPTE0_FULL,
517	VMX_VMCS64_GUEST_PDPTE0_HIGH,
518	VMX_VMCS64_GUEST_PDPTE1_FULL,
519	VMX_VMCS64_GUEST_PDPTE1_HIGH,
520	VMX_VMCS64_GUEST_PDPTE2_FULL,
521	VMX_VMCS64_GUEST_PDPTE2_HIGH,
522	VMX_VMCS64_GUEST_PDPTE3_FULL,
523	VMX_VMCS64_GUEST_PDPTE3_HIGH,
524	VMX_VMCS64_GUEST_BNDCFGS_FULL,
525	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
526
527	/* 64-bit host-state fields. */
528	VMX_VMCS64_HOST_PAT_FULL,
529	VMX_VMCS64_HOST_PAT_HIGH,
530	VMX_VMCS64_HOST_EFER_FULL,
531	VMX_VMCS64_HOST_EFER_HIGH,
532	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
533	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
534
535	/* 32-bit control fields. */
536	VMX_VMCS32_CTRL_PIN_EXEC,
537	VMX_VMCS32_CTRL_PROC_EXEC,
538	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
539	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
540	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
541	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
542	VMX_VMCS32_CTRL_EXIT,
543	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
544	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
545	VMX_VMCS32_CTRL_ENTRY,
546	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
547	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
548	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
549	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
550	VMX_VMCS32_CTRL_TPR_THRESHOLD,
551	VMX_VMCS32_CTRL_PROC_EXEC2,
552	VMX_VMCS32_CTRL_PLE_GAP,
553	VMX_VMCS32_CTRL_PLE_WINDOW,
554
555	/* 32-bits read-only fields. */
556	VMX_VMCS32_RO_VM_INSTR_ERROR,
557	VMX_VMCS32_RO_EXIT_REASON,
558	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
559	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
560	VMX_VMCS32_RO_IDT_VECTORING_INFO,
561	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
562	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
563	VMX_VMCS32_RO_EXIT_INSTR_INFO,
564
565	/* 32-bit guest-state fields. */
566	VMX_VMCS32_GUEST_ES_LIMIT,
567	VMX_VMCS32_GUEST_CS_LIMIT,
568	VMX_VMCS32_GUEST_SS_LIMIT,
569	VMX_VMCS32_GUEST_DS_LIMIT,
570	VMX_VMCS32_GUEST_FS_LIMIT,
571	VMX_VMCS32_GUEST_GS_LIMIT,
572	VMX_VMCS32_GUEST_LDTR_LIMIT,
573	VMX_VMCS32_GUEST_TR_LIMIT,
574	VMX_VMCS32_GUEST_GDTR_LIMIT,
575	VMX_VMCS32_GUEST_IDTR_LIMIT,
576	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
577	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
578	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
579	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
580	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
581	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
582	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
583	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
584	VMX_VMCS32_GUEST_INT_STATE,
585	VMX_VMCS32_GUEST_ACTIVITY_STATE,
586	VMX_VMCS32_GUEST_SMBASE,
587	VMX_VMCS32_GUEST_SYSENTER_CS,
588	VMX_VMCS32_PREEMPT_TIMER_VALUE,
589
590	/* 32-bit host-state fields. */
591	VMX_VMCS32_HOST_SYSENTER_CS,
592
593	/* Natural-width control fields. */
594	VMX_VMCS_CTRL_CR0_MASK,
595	VMX_VMCS_CTRL_CR4_MASK,
596	VMX_VMCS_CTRL_CR0_READ_SHADOW,
597	VMX_VMCS_CTRL_CR4_READ_SHADOW,
598	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
599	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
600	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
601	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
602
603	/* Natural-width read-only data fields. */
604	VMX_VMCS_RO_EXIT_QUALIFICATION,
605	VMX_VMCS_RO_IO_RCX,
606	VMX_VMCS_RO_IO_RSI,
607	VMX_VMCS_RO_IO_RDI,
608	VMX_VMCS_RO_IO_RIP,
609	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
610
611	/* Natural-width guest-state field */
612	VMX_VMCS_GUEST_CR0,
613	VMX_VMCS_GUEST_CR3,
614	VMX_VMCS_GUEST_CR4,
615	VMX_VMCS_GUEST_ES_BASE,
616	VMX_VMCS_GUEST_CS_BASE,
617	VMX_VMCS_GUEST_SS_BASE,
618	VMX_VMCS_GUEST_DS_BASE,
619	VMX_VMCS_GUEST_FS_BASE,
620	VMX_VMCS_GUEST_GS_BASE,
621	VMX_VMCS_GUEST_LDTR_BASE,
622	VMX_VMCS_GUEST_TR_BASE,
623	VMX_VMCS_GUEST_GDTR_BASE,
624	VMX_VMCS_GUEST_IDTR_BASE,
625	VMX_VMCS_GUEST_DR7,
626	VMX_VMCS_GUEST_RSP,
627	VMX_VMCS_GUEST_RIP,
628	VMX_VMCS_GUEST_RFLAGS,
629	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
630	VMX_VMCS_GUEST_SYSENTER_ESP,
631	VMX_VMCS_GUEST_SYSENTER_EIP,
632
633	/* Natural-width host-state fields */
634	VMX_VMCS_HOST_CR0,
635	VMX_VMCS_HOST_CR3,
636	VMX_VMCS_HOST_CR4,
637	VMX_VMCS_HOST_FS_BASE,
638	VMX_VMCS_HOST_GS_BASE,
639	VMX_VMCS_HOST_TR_BASE,
640	VMX_VMCS_HOST_GDTR_BASE,
641	VMX_VMCS_HOST_IDTR_BASE,
642	VMX_VMCS_HOST_SYSENTER_ESP,
643	VMX_VMCS_HOST_SYSENTER_EIP,
644	VMX_VMCS_HOST_RSP,
645	VMX_VMCS_HOST_RIP
646	};
647	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
648
649	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
650	static const uint32_t g_aVmcsCacheSegBase[] =
651	{
652	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
653	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
654	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
655	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
656	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
657	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
658	};
659	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
660	#endif
661	static const uint32_t g_aVmcsSegBase[] =
662	{
663	VMX_VMCS_GUEST_ES_BASE,
664	VMX_VMCS_GUEST_CS_BASE,
665	VMX_VMCS_GUEST_SS_BASE,
666	VMX_VMCS_GUEST_DS_BASE,
667	VMX_VMCS_GUEST_FS_BASE,
668	VMX_VMCS_GUEST_GS_BASE
669	};
670	static const uint32_t g_aVmcsSegSel[] =
671	{
672	VMX_VMCS16_GUEST_ES_SEL,
673	VMX_VMCS16_GUEST_CS_SEL,
674	VMX_VMCS16_GUEST_SS_SEL,
675	VMX_VMCS16_GUEST_DS_SEL,
676	VMX_VMCS16_GUEST_FS_SEL,
677	VMX_VMCS16_GUEST_GS_SEL
678	};
679	static const uint32_t g_aVmcsSegLimit[] =
680	{
681	VMX_VMCS32_GUEST_ES_LIMIT,
682	VMX_VMCS32_GUEST_CS_LIMIT,
683	VMX_VMCS32_GUEST_SS_LIMIT,
684	VMX_VMCS32_GUEST_DS_LIMIT,
685	VMX_VMCS32_GUEST_FS_LIMIT,
686	VMX_VMCS32_GUEST_GS_LIMIT
687	};
688	static const uint32_t g_aVmcsSegAttr[] =
689	{
690	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
691	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
692	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
693	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
694	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
695	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
696	};
697	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
698	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
699	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
700	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
701
702	#ifdef HMVMX_USE_FUNCTION_TABLE
703	/**
704	* VMX_EXIT dispatch table.
705	*/
706	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
707	{
708	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
709	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
710	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
711	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
712	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
713	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
714	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
715	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
716	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
717	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
718	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
719	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
720	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
721	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
722	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
723	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
724	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
725	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
726	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
727	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
728	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
729	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
730	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
731	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
732	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
733	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
734	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
735	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
736	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
737	#else
738	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
739	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
740	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
741	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
742	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
743	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
744	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
745	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
746	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
747	#endif
748	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
749	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
750	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
751	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
752	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
753	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
754	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
755	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
756	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
757	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
758	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
759	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
760	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
761	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
762	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
763	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
764	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
765	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
766	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
767	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
768	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
769	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
770	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
771	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
772	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
773	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
774	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
775	#else
776	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
777	#endif
778	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
779	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
780	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
781	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
782	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
783	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
784	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
785	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
786	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
787	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
788	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
789	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
790	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
791	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
792	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
793	};
794	#endif /* HMVMX_USE_FUNCTION_TABLE */
795
796	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
797	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
798	{
799	/* 0 */ "(Not Used)",
800	/* 1 */ "VMCALL executed in VMX root operation.",
801	/* 2 */ "VMCLEAR with invalid physical address.",
802	/* 3 */ "VMCLEAR with VMXON pointer.",
803	/* 4 */ "VMLAUNCH with non-clear VMCS.",
804	/* 5 */ "VMRESUME with non-launched VMCS.",
805	/* 6 */ "VMRESUME after VMXOFF",
806	/* 7 */ "VM-entry with invalid control fields.",
807	/* 8 */ "VM-entry with invalid host state fields.",
808	/* 9 */ "VMPTRLD with invalid physical address.",
809	/* 10 */ "VMPTRLD with VMXON pointer.",
810	/* 11 */ "VMPTRLD with incorrect revision identifier.",
811	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
812	/* 13 */ "VMWRITE to read-only VMCS component.",
813	/* 14 */ "(Not Used)",
814	/* 15 */ "VMXON executed in VMX root operation.",
815	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
816	/* 17 */ "VM-entry with non-launched executing VMCS.",
817	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
818	/* 19 */ "VMCALL with non-clear VMCS.",
819	/* 20 */ "VMCALL with invalid VM-exit control fields.",
820	/* 21 */ "(Not Used)",
821	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
822	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
823	/* 24 */ "VMCALL with invalid SMM-monitor features.",
824	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
825	/* 26 */ "VM-entry with events blocked by MOV SS.",
826	/* 27 */ "(Not Used)",
827	/* 28 */ "Invalid operand to INVEPT/INVVPID."
828	};
829	#endif /* VBOX_STRICT && LOG_ENABLED */
830
831
832	/**
833	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
834	*
835	* Any bit set in this mask is owned by the host/hypervisor and would cause a
836	* VM-exit when modified by the guest.
837	*
838	* @returns The static CR0 guest/host mask.
839	* @param pVCpu The cross context virtual CPU structure.
840	*/
841	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
842	{
843	/*
844	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
845	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
846	*/
847	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
848	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
849	* and @bugref{6944}. */
850	PVM pVM = pVCpu->CTX_SUFF(pVM);
851	return ( X86_CR0_PE
852	\| X86_CR0_NE
853	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
854	\| X86_CR0_PG
855	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
856	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
857	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
858	}
859
860
861	/**
862	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
863	*
864	* Any bit set in this mask is owned by the host/hypervisor and would cause a
865	* VM-exit when modified by the guest.
866	*
867	* @returns The static CR4 guest/host mask.
868	* @param pVCpu The cross context virtual CPU structure.
869	*/
870	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
871	{
872	/*
873	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
874	* these bits are reserved on hardware that does not support them. Since the
875	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
876	* these bits and handle it depending on whether we expose them to the guest.
877	*/
878	PVM pVM = pVCpu->CTX_SUFF(pVM);
879	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
880	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
881	return ( X86_CR4_VMXE
882	\| X86_CR4_VME
883	\| X86_CR4_PAE
884	\| X86_CR4_PGE
885	\| X86_CR4_PSE
886	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
887	\| (fPcid ? X86_CR4_PCIDE : 0));
888	}
889
890
891	/**
892	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
893	* area.
894	*
895	* @returns @c true if it's different, @c false otherwise.
896	* @param pVmcsInfo The VMCS info. object.
897	*/
898	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
899	{
900	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
901	&& pVmcsInfo->pvGuestMsrStore);
902	}
903
904
905	/**
906	* Checks whether one of the given Pin-based VM-execution controls are set.
907	*
908	* @returns @c true if set, @c false otherwise.
909	* @param pVCpu The cross context virtual CPU structure.
910	* @param pVmxTransient The VMX-transient structure.
911	* @param uPinCtls The Pin-based VM-execution controls to check.
912	*
913	* @remarks This will not check merged controls when executing a nested-guest
914	* but the original control specified by the guest hypervisor.
915	*/
916	static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
917	{
918	if (!pVmxTransient->fIsNestedGuest)
919	{
920	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
921	return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
922	}
923	return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
924	}
925
926
927	/**
928	* Sets the given Processor-based VM-execution controls.
929	*
930	* @param pVmxTransient The VMX-transient structure.
931	* @param uProcCtls The Processor-based VM-execution controls to set.
932	*/
933	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
934	{
935	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
936	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
937	{
938	pVmcsInfo->u32ProcCtls \|= uProcCtls;
939	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
940	AssertRC(rc);
941	}
942	}
943
944
945	/**
946	* Removes the given Processor-based VM-execution controls.
947	*
948	* @param pVCpu The cross context virtual CPU structure.
949	* @param pVmxTransient The VMX-transient structure.
950	* @param uProcCtls The Processor-based VM-execution controls to remove.
951	*
952	* @remarks When executing a nested-guest, this will not remove any of the specified
953	* controls if the guest hypervisor has set any one of them.
954	*/
955	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
956	{
957	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
958	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
959	? true
960	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT);
961	#else
962	NOREF(pVCpu);
963	bool const fRemoveCtls = true;
964	#endif
965	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
966	if ( fRemoveCtls
967	&& (pVmcsInfo->u32ProcCtls & uProcCtls))
968	{
969	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
970	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
971	AssertRC(rc);
972	}
973	}
974
975
976	/**
977	* Sets the TSC offset for the current VMCS.
978	*
979	* @param pVCpu The cross context virtual CPU structure.
980	* @param uTscOffset The TSC offset to set.
981	* @param pVmcsInfo The VMCS info. object.
982	*/
983	static void hmR0VmxSetTscOffsetVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
984	{
985	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
986	if (pVmcsInfo->u64TscOffset != uTscOffset)
987	{
988	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
989	AssertRC(rc);
990	pVmcsInfo->u64TscOffset = uTscOffset;
991	}
992	}
993
994
995	/**
996	* Adds one or more exceptions to the exception bitmap and commits it to the current
997	* VMCS.
998	*
999	* @returns VBox status code.
1000	* @param pVmxTransient The VMX-transient structure.
1001	* @param uXcptMask The exception(s) to add.
1002	*/
1003	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1004	{
1005	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1006	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1007	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1008	{
1009	uXcptBitmap \|= uXcptMask;
1010	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1011	AssertRCReturn(rc, rc);
1012	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1013	}
1014	return VINF_SUCCESS;
1015	}
1016
1017
1018	/**
1019	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1020	*
1021	* @returns VBox status code.
1022	* @param pVmxTransient The VMX-transient structure.
1023	* @param uXcpt The exception to add.
1024	*/
1025	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1026	{
1027	Assert(uXcpt <= X86_XCPT_LAST);
1028	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1029	}
1030
1031
1032	/**
1033	* Remove one or more exceptions from the exception bitmap and commits it to the
1034	* current VMCS.
1035	*
1036	* This takes care of not removing the exception intercept if a nested-guest
1037	* requires the exception to be intercepted.
1038	*
1039	* @returns VBox status code.
1040	* @param pVCpu The cross context virtual CPU structure.
1041	* @param pVmxTransient The VMX-transient structure.
1042	* @param uXcptMask The exception(s) to remove.
1043	*/
1044	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1045	{
1046	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1047	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1048	if (u32XcptBitmap & uXcptMask)
1049	{
1050	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1051	if (!pVmxTransient->fIsNestedGuest)
1052	{ /* likely */ }
1053	else
1054	{
1055	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1056	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1057	}
1058	#endif
1059	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1060	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1061	\| RT_BIT(X86_XCPT_DE)
1062	\| RT_BIT(X86_XCPT_NM)
1063	\| RT_BIT(X86_XCPT_TS)
1064	\| RT_BIT(X86_XCPT_UD)
1065	\| RT_BIT(X86_XCPT_NP)
1066	\| RT_BIT(X86_XCPT_SS)
1067	\| RT_BIT(X86_XCPT_GP)
1068	\| RT_BIT(X86_XCPT_PF)
1069	\| RT_BIT(X86_XCPT_MF));
1070	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1071	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1072	#endif
1073	if (uXcptMask)
1074	{
1075	/* Validate we are not removing any essential exception intercepts. */
1076	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1077	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1078	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1079
1080	/* Remove it from the exception bitmap. */
1081	u32XcptBitmap &= ~uXcptMask;
1082
1083	/* Commit and update the cache if necessary. */
1084	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1085	{
1086	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1087	AssertRCReturn(rc, rc);
1088	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1089	}
1090	}
1091	}
1092	return VINF_SUCCESS;
1093	}
1094
1095
1096	/**
1097	* Remove an exceptions from the exception bitmap and commits it to the current
1098	* VMCS.
1099	*
1100	* @returns VBox status code.
1101	* @param pVCpu The cross context virtual CPU structure.
1102	* @param pVmxTransient The VMX-transient structure.
1103	* @param uXcpt The exception to remove.
1104	*/
1105	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1106	{
1107	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1108	}
1109
1110
1111	/**
1112	* Loads the VMCS specified by the VMCS info. object.
1113	*
1114	* @returns VBox status code.
1115	* @param pVmcsInfo The VMCS info. object.
1116	*
1117	* @remarks Can be called with interrupts disabled.
1118	*/
1119	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1120	{
1121	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1123
1124	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1125	if (RT_SUCCESS(rc))
1126	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1127	return rc;
1128	}
1129
1130
1131	/**
1132	* Clears the VMCS specified by the VMCS info. object.
1133	*
1134	* @returns VBox status code.
1135	* @param pVmcsInfo The VMCS info. object.
1136	*
1137	* @remarks Can be called with interrupts disabled.
1138	*/
1139	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1140	{
1141	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1142	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1143
1144	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1145	if (RT_SUCCESS(rc))
1146	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1147	return rc;
1148	}
1149
1150
1151	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1152	/**
1153	* Loads the shadow VMCS specified by the VMCS info. object.
1154	*
1155	* @returns VBox status code.
1156	* @param pVmcsInfo The VMCS info. object.
1157	*
1158	* @remarks Can be called with interrupts disabled.
1159	*/
1160	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1161	{
1162	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1163	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1164
1165	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1166	if (RT_SUCCESS(rc))
1167	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1168	return rc;
1169	}
1170
1171
1172	/**
1173	* Clears the shadow VMCS specified by the VMCS info. object.
1174	*
1175	* @returns VBox status code.
1176	* @param pVmcsInfo The VMCS info. object.
1177	*
1178	* @remarks Can be called with interrupts disabled.
1179	*/
1180	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1181	{
1182	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1183	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1184
1185	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1186	if (RT_SUCCESS(rc))
1187	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1188	return rc;
1189	}
1190
1191
1192	/**
1193	* Switches from and to the specified VMCSes.
1194	*
1195	* @returns VBox status code.
1196	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1197	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1198	*
1199	* @remarks Called with interrupts disabled.
1200	*/
1201	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1202	{
1203	/*
1204	* Clear the VMCS we are switching out if it has not already been cleared.
1205	* This will sync any CPU internal data back to the VMCS.
1206	*/
1207	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1208	{
1209	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1210	if (RT_SUCCESS(rc))
1211	{
1212	/*
1213	* The shadow VMCS, if any, would not be active at this point since we
1214	* would have cleared it while importing the virtual hardware-virtualization
1215	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1216	* clear the shadow VMCS here, just assert for safety.
1217	*/
1218	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1219	}
1220	else
1221	return rc;
1222	}
1223
1224	/*
1225	* Clear the VMCS we are switching to if it has not already been cleared.
1226	* This will initialize the VMCS launch state to "clear" required for loading it.
1227	*
1228	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1229	*/
1230	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1231	{
1232	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1233	if (RT_SUCCESS(rc))
1234	{ /* likely */ }
1235	else
1236	return rc;
1237	}
1238
1239	/*
1240	* Finally, load the VMCS we are switching to.
1241	*/
1242	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1243	}
1244
1245
1246	/**
1247	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1248	* caller.
1249	*
1250	* @returns VBox status code.
1251	* @param pVCpu The cross context virtual CPU structure.
1252	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1253	* true) or guest VMCS (pass false).
1254	*/
1255	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPU pVCpu, bool fSwitchToNstGstVmcs)
1256	{
1257	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1258	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1259
1260	PVMXVMCSINFO pVmcsInfoFrom;
1261	PVMXVMCSINFO pVmcsInfoTo;
1262	if (fSwitchToNstGstVmcs)
1263	{
1264	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1265	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1266	}
1267	else
1268	{
1269	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1270	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1271	}
1272
1273	/*
1274	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1275	* preemption hook code path acquires the current VMCS.
1276	*/
1277	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1278
1279	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1280	if (RT_SUCCESS(rc))
1281	{
1282	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1283
1284	/*
1285	* If we are switching to a VMCS that was executed on a different host CPU or was
1286	* never executed before, flag that we need to export the host state before executing
1287	* guest/nested-guest code using hardware-assisted VMX.
1288	*
1289	* This could probably be done in a preemptible context since the preemption hook
1290	* will flag the necessary change in host context. However, since preemption is
1291	* already disabled and to avoid making assumptions about host specific code in
1292	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1293	* disabled.
1294	*/
1295	if (pVmcsInfoTo->idHostCpu == RTMpCpuId())
1296	{ /* likely */ }
1297	else
1298	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1299
1300	ASMSetFlags(fEFlags);
1301
1302	/*
1303	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1304	* flag that we need to update the host MSR values there. Even if we decide in the
1305	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1306	* if its content differs, we would have to update the host MSRs anyway.
1307	*/
1308	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1309	}
1310	else
1311	ASMSetFlags(fEFlags);
1312	return rc;
1313	}
1314	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1315
1316
1317	/**
1318	* Updates the VM's last error record.
1319	*
1320	* If there was a VMX instruction error, reads the error data from the VMCS and
1321	* updates VCPU's last error record as well.
1322	*
1323	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1324	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1325	* VERR_VMX_INVALID_VMCS_FIELD.
1326	* @param rc The error code.
1327	*/
1328	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1329	{
1330	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1331	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1332	{
1333	AssertPtrReturnVoid(pVCpu);
1334	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1335	}
1336	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1337	}
1338
1339
1340	#ifdef VBOX_STRICT
1341	/**
1342	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1343	* transient structure.
1344	*
1345	* @returns VBox status code.
1346	* @param pVmxTransient The VMX-transient structure.
1347	*
1348	* @remarks No-long-jump zone!!!
1349	*/
1350	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1351	{
1352	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1353	AssertRCReturn(rc, rc);
1354	return VINF_SUCCESS;
1355	}
1356
1357
1358	/**
1359	* Reads the VM-entry exception error code field from the VMCS into
1360	* the VMX transient structure.
1361	*
1362	* @returns VBox status code.
1363	* @param pVmxTransient The VMX-transient structure.
1364	*
1365	* @remarks No-long-jump zone!!!
1366	*/
1367	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1368	{
1369	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1370	AssertRCReturn(rc, rc);
1371	return VINF_SUCCESS;
1372	}
1373
1374
1375	/**
1376	* Reads the VM-entry exception error code field from the VMCS into
1377	* the VMX transient structure.
1378	*
1379	* @returns VBox status code.
1380	* @param pVmxTransient The VMX-transient structure.
1381	*
1382	* @remarks No-long-jump zone!!!
1383	*/
1384	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1385	{
1386	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1387	AssertRCReturn(rc, rc);
1388	return VINF_SUCCESS;
1389	}
1390	#endif /* VBOX_STRICT */
1391
1392
1393	/**
1394	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1395	* transient structure.
1396	*
1397	* @returns VBox status code.
1398	* @param pVmxTransient The VMX-transient structure.
1399	*/
1400	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1401	{
1402	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1403	{
1404	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1405	AssertRCReturn(rc,rc);
1406	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1407	}
1408	return VINF_SUCCESS;
1409	}
1410
1411
1412	/**
1413	* Reads the VM-exit interruption error code from the VMCS into the VMX
1414	* transient structure.
1415	*
1416	* @returns VBox status code.
1417	* @param pVmxTransient The VMX-transient structure.
1418	*/
1419	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1420	{
1421	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1422	{
1423	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1424	AssertRCReturn(rc, rc);
1425	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1426	}
1427	return VINF_SUCCESS;
1428	}
1429
1430
1431	/**
1432	* Reads the VM-exit instruction length field from the VMCS into the VMX
1433	* transient structure.
1434	*
1435	* @returns VBox status code.
1436	* @param pVmxTransient The VMX-transient structure.
1437	*/
1438	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1439	{
1440	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1441	{
1442	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1443	AssertRCReturn(rc, rc);
1444	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1445	}
1446	return VINF_SUCCESS;
1447	}
1448
1449
1450	/**
1451	* Reads the VM-exit instruction-information field from the VMCS into
1452	* the VMX transient structure.
1453	*
1454	* @returns VBox status code.
1455	* @param pVmxTransient The VMX-transient structure.
1456	*/
1457	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1458	{
1459	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1460	{
1461	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1462	AssertRCReturn(rc, rc);
1463	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1464	}
1465	return VINF_SUCCESS;
1466	}
1467
1468
1469	/**
1470	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1471	*
1472	* @returns VBox status code.
1473	* @param pVCpu The cross context virtual CPU structure of the
1474	* calling EMT. (Required for the VMCS cache case.)
1475	* @param pVmxTransient The VMX-transient structure.
1476	*/
1477	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1478	{
1479	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1480	{
1481	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1482	AssertRCReturn(rc, rc);
1483	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1484	}
1485	return VINF_SUCCESS;
1486	}
1487
1488
1489	/**
1490	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1491	*
1492	* @returns VBox status code.
1493	* @param pVCpu The cross context virtual CPU structure of the
1494	* calling EMT. (Required for the VMCS cache case.)
1495	* @param pVmxTransient The VMX-transient structure.
1496	*/
1497	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1498	{
1499	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1500	{
1501	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1502	AssertRCReturn(rc, rc);
1503	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1504	}
1505	return VINF_SUCCESS;
1506	}
1507
1508
1509	/**
1510	* Reads the IDT-vectoring information field from the VMCS into the VMX
1511	* transient structure.
1512	*
1513	* @returns VBox status code.
1514	* @param pVmxTransient The VMX-transient structure.
1515	*
1516	* @remarks No-long-jump zone!!!
1517	*/
1518	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1519	{
1520	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1521	{
1522	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1523	AssertRCReturn(rc, rc);
1524	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1525	}
1526	return VINF_SUCCESS;
1527	}
1528
1529
1530	/**
1531	* Reads the IDT-vectoring error code from the VMCS into the VMX
1532	* transient structure.
1533	*
1534	* @returns VBox status code.
1535	* @param pVmxTransient The VMX-transient structure.
1536	*/
1537	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1538	{
1539	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1540	{
1541	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1542	AssertRCReturn(rc, rc);
1543	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1544	}
1545	return VINF_SUCCESS;
1546	}
1547
1548
1549	/**
1550	* Enters VMX root mode operation on the current CPU.
1551	*
1552	* @returns VBox status code.
1553	* @param pVM The cross context VM structure. Can be
1554	* NULL, after a resume.
1555	* @param HCPhysCpuPage Physical address of the VMXON region.
1556	* @param pvCpuPage Pointer to the VMXON region.
1557	*/
1558	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1559	{
1560	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1561	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1562	Assert(pvCpuPage);
1563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564
1565	if (pVM)
1566	{
1567	/* Write the VMCS revision identifier to the VMXON region. */
1568	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1569	}
1570
1571	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1572	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1573
1574	/* Enable the VMX bit in CR4 if necessary. */
1575	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1576
1577	/* Enter VMX root mode. */
1578	int rc = VMXEnable(HCPhysCpuPage);
1579	if (RT_FAILURE(rc))
1580	{
1581	if (!(uOldCr4 & X86_CR4_VMXE))
1582	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1583
1584	if (pVM)
1585	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1586	}
1587
1588	/* Restore interrupts. */
1589	ASMSetFlags(fEFlags);
1590	return rc;
1591	}
1592
1593
1594	/**
1595	* Exits VMX root mode operation on the current CPU.
1596	*
1597	* @returns VBox status code.
1598	*/
1599	static int hmR0VmxLeaveRootMode(void)
1600	{
1601	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1602
1603	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1604	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1605
1606	/* If we're for some reason not in VMX root mode, then don't leave it. */
1607	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1608
1609	int rc;
1610	if (uHostCr4 & X86_CR4_VMXE)
1611	{
1612	/* Exit VMX root mode and clear the VMX bit in CR4. */
1613	VMXDisable();
1614	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1615	rc = VINF_SUCCESS;
1616	}
1617	else
1618	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1619
1620	/* Restore interrupts. */
1621	ASMSetFlags(fEFlags);
1622	return rc;
1623	}
1624
1625
1626	/**
1627	* Allocates and maps a physically contiguous page. The allocated page is
1628	* zero'd out (used by various VT-x structures).
1629	*
1630	* @returns IPRT status code.
1631	* @param pMemObj Pointer to the ring-0 memory object.
1632	* @param ppVirt Where to store the virtual address of the allocation.
1633	* @param pHCPhys Where to store the physical address of the allocation.
1634	*/
1635	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1636	{
1637	AssertPtr(pMemObj);
1638	AssertPtr(ppVirt);
1639	AssertPtr(pHCPhys);
1640	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1641	if (RT_FAILURE(rc))
1642	return rc;
1643	ppVirt = RTR0MemObjAddress(pMemObj);
1644	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1645	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1646	return VINF_SUCCESS;
1647	}
1648
1649
1650	/**
1651	* Frees and unmaps an allocated, physical page.
1652	*
1653	* @param pMemObj Pointer to the ring-0 memory object.
1654	* @param ppVirt Where to re-initialize the virtual address of allocation as
1655	* 0.
1656	* @param pHCPhys Where to re-initialize the physical address of the
1657	* allocation as 0.
1658	*/
1659	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1660	{
1661	AssertPtr(pMemObj);
1662	AssertPtr(ppVirt);
1663	AssertPtr(pHCPhys);
1664	/* NULL is valid, accepted and ignored by the free function below. */
1665	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1666	*pMemObj = NIL_RTR0MEMOBJ;
1667	*ppVirt = NULL;
1668	*pHCPhys = NIL_RTHCPHYS;
1669	}
1670
1671
1672	/**
1673	* Initializes a VMCS info. object.
1674	*
1675	* @param pVmcsInfo The VMCS info. object.
1676	*/
1677	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1678	{
1679	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1680
1681	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1682	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1683	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1684	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1685	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1686	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1687	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1688	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1689	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1690	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1691	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1692	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1693	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1694	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1695	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1696	pVmcsInfo->idHostCpu = NIL_RTCPUID;
1697	}
1698
1699
1700	/**
1701	* Frees the VT-x structures for a VMCS info. object.
1702	*
1703	* @param pVM The cross context VM structure.
1704	* @param pVmcsInfo The VMCS info. object.
1705	*/
1706	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1707	{
1708	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1709
1710	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1711	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1712	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1713	#endif
1714
1715	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1716	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1717
1718	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1719	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1720	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1721
1722	hmR0VmxInitVmcsInfo(pVmcsInfo);
1723	}
1724
1725
1726	/**
1727	* Allocates the VT-x structures for a VMCS info. object.
1728	*
1729	* @returns VBox status code.
1730	* @param pVCpu The cross context virtual CPU structure.
1731	* @param pVmcsInfo The VMCS info. object.
1732	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1733	*/
1734	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1735	{
1736	PVM pVM = pVCpu->CTX_SUFF(pVM);
1737
1738	/* Allocate the guest VM control structure (VMCS). */
1739	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1740	if (RT_SUCCESS(rc))
1741	{
1742	if (!fIsNstGstVmcs)
1743	{
1744	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1745	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1746	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1747	#endif
1748	if (RT_SUCCESS(rc))
1749	{
1750	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1751	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1752	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1753	{
1754	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1755	NULL /* pR3Ptr /, NULL / pRCPtr */);
1756	}
1757	}
1758	}
1759	else
1760	{
1761	/* We don't yet support exposing VMCS shadowing to the guest. */
1762	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1763	Assert(!pVmcsInfo->pvShadowVmcs);
1764
1765	/* Get the allocated virtual-APIC page from CPUM. */
1766	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1767	{
1768	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1769	&pVmcsInfo->HCPhysVirtApic);
1770	Assert(pVmcsInfo->pbVirtApic);
1771	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1772	}
1773	}
1774
1775	if (RT_SUCCESS(rc))
1776	{
1777	/*
1778	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1779	* transparent accesses of specific MSRs.
1780	*
1781	* If the condition for enabling MSR bitmaps changes here, don't forget to
1782	* update HMIsMsrBitmapActive().
1783	*
1784	* We don't share MSR bitmaps between the guest and nested-guest as we then
1785	* don't need to care about carefully restoring the guest MSR bitmap.
1786	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1787	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1788	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1789	* we do that later while merging VMCS.
1790	*/
1791	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1792	{
1793	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1794	if ( RT_SUCCESS(rc)
1795	&& !fIsNstGstVmcs)
1796	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1797	}
1798
1799	if (RT_SUCCESS(rc))
1800	{
1801	/*
1802	* Allocate the VM-entry MSR-load area for the guest MSRs.
1803	*
1804	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1805	* the guest and nested-guest.
1806	*/
1807	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1808	&pVmcsInfo->HCPhysGuestMsrLoad);
1809	if (RT_SUCCESS(rc))
1810	{
1811	/*
1812	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1813	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1814	*/
1815	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1816	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1817	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1818
1819	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1820	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1821	&pVmcsInfo->HCPhysHostMsrLoad);
1822	}
1823	}
1824	}
1825	}
1826
1827	return rc;
1828	}
1829
1830
1831	/**
1832	* Free all VT-x structures for the VM.
1833	*
1834	* @returns IPRT status code.
1835	* @param pVM The cross context VM structure.
1836	*/
1837	static void hmR0VmxStructsFree(PVM pVM)
1838	{
1839	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1840	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1841	#endif
1842	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1843
1844	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1845	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1846	{
1847	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1848	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1849	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1850	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1851	}
1852	#endif
1853
1854	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1855	{
1856	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1857	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1858	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1859	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1860	if (pVM->cpum.ro.GuestFeatures.fVmx)
1861	{
1862	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1863	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1864	}
1865	#endif
1866	}
1867	}
1868
1869
1870	/**
1871	* Allocate all VT-x structures for the VM.
1872	*
1873	* @returns IPRT status code.
1874	* @param pVM The cross context VM structure.
1875	*/
1876	static int hmR0VmxStructsAlloc(PVM pVM)
1877	{
1878	/*
1879	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1880	* The VMCS size cannot be more than 4096 bytes.
1881	*
1882	* See Intel spec. Appendix A.1 "Basic VMX Information".
1883	*/
1884	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1885	if (cbVmcs <= X86_PAGE_4K_SIZE)
1886	{ /* likely */ }
1887	else
1888	{
1889	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1890	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1891	}
1892
1893	/*
1894	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1895	*/
1896	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1897	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1898	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1899	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1900	#endif
1901
1902	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1903	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1904	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1905
1906	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1907	{
1908	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1909	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1910	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1911	}
1912
1913	/*
1914	* Allocate per-VM VT-x structures.
1915	*/
1916	int rc = VINF_SUCCESS;
1917	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1918	/* Allocate crash-dump magic scratch page. */
1919	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1920	if (RT_FAILURE(rc))
1921	{
1922	hmR0VmxStructsFree(pVM);
1923	return rc;
1924	}
1925	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1926	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1927	#endif
1928
1929	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1930	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1931	{
1932	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1933	&pVM->hm.s.vmx.HCPhysApicAccess);
1934	if (RT_FAILURE(rc))
1935	{
1936	hmR0VmxStructsFree(pVM);
1937	return rc;
1938	}
1939	}
1940
1941	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1942	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1943	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1944	{
1945	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1946	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1947	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1948	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1949	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1950	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1951	{
1952	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1953	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1954	if (RT_SUCCESS(rc))
1955	{
1956	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1957	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1958	}
1959	}
1960	else
1961	rc = VERR_NO_MEMORY;
1962
1963	if (RT_FAILURE(rc))
1964	{
1965	hmR0VmxStructsFree(pVM);
1966	return rc;
1967	}
1968	}
1969	#endif
1970
1971	/*
1972	* Initialize per-VCPU VT-x structures.
1973	*/
1974	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1975	{
1976	/* Allocate the guest VMCS structures. */
1977	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1978	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1979	if (RT_SUCCESS(rc))
1980	{
1981	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1982	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1983	if (pVM->cpum.ro.GuestFeatures.fVmx)
1984	{
1985	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1986	if (RT_SUCCESS(rc))
1987	{ /* likely */ }
1988	else
1989	break;
1990	}
1991	#endif
1992	}
1993	else
1994	break;
1995	}
1996
1997	if (RT_FAILURE(rc))
1998	{
1999	hmR0VmxStructsFree(pVM);
2000	return rc;
2001	}
2002
2003	return VINF_SUCCESS;
2004	}
2005
2006	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2007	/**
2008	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2009	*
2010	* @returns @c true if the MSR is intercepted, @c false otherwise.
2011	* @param pvMsrBitmap The MSR bitmap.
2012	* @param offMsr The MSR byte offset.
2013	* @param iBit The bit offset from the byte offset.
2014	*/
2015	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2016	{
2017	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2018	Assert(pbMsrBitmap);
2019	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2020	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2021	}
2022	#endif
2023
2024	/**
2025	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2026	*
2027	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2028	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2029	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2030	* the read/write access of this MSR.
2031	*
2032	* @param pVCpu The cross context virtual CPU structure.
2033	* @param pVmcsInfo The VMCS info. object.
2034	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2035	* @param idMsr The MSR value.
2036	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2037	* include both a read -and- a write permission!
2038	*
2039	* @sa CPUMGetVmxMsrPermission.
2040	* @remarks Can be called with interrupts disabled.
2041	*/
2042	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2043	{
2044	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2045	Assert(pbMsrBitmap);
2046	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2047
2048	/*
2049	* MSR-bitmap Layout:
2050	* Byte index MSR range Interpreted as
2051	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2052	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2053	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2054	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2055	*
2056	* A bit corresponding to an MSR within the above range causes a VM-exit
2057	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2058	* the MSR range, it always cause a VM-exit.
2059	*
2060	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2061	*/
2062	uint16_t const offBitmapRead = 0;
2063	uint16_t const offBitmapWrite = 0x800;
2064	uint16_t offMsr;
2065	int32_t iBit;
2066	if (idMsr <= UINT32_C(0x00001fff))
2067	{
2068	offMsr = 0;
2069	iBit = idMsr;
2070	}
2071	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2072	{
2073	offMsr = 0x400;
2074	iBit = idMsr - UINT32_C(0xc0000000);
2075	}
2076	else
2077	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2078
2079	/*
2080	* Set the MSR read permission.
2081	*/
2082	uint16_t const offMsrRead = offBitmapRead + offMsr;
2083	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2084	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2085	{
2086	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2087	bool const fClear = !fIsNstGstVmcs ? true
2088	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2089	#else
2090	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2091	bool const fClear = true;
2092	#endif
2093	if (fClear)
2094	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2095	}
2096	else
2097	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2098
2099	/*
2100	* Set the MSR write permission.
2101	*/
2102	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2103	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2104	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2105	{
2106	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2107	bool const fClear = !fIsNstGstVmcs ? true
2108	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2109	#else
2110	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2111	bool const fClear = true;
2112	#endif
2113	if (fClear)
2114	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2115	}
2116	else
2117	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2118	}
2119
2120
2121	/**
2122	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2123	* area.
2124	*
2125	* @returns VBox status code.
2126	* @param pVCpu The cross context virtual CPU structure.
2127	* @param pVmcsInfo The VMCS info. object.
2128	* @param cMsrs The number of MSRs.
2129	*/
2130	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2131	{
2132	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2133	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2134	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2135	{
2136	/* Commit the MSR counts to the VMCS and update the cache. */
2137	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2138	{
2139	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2140	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2141	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2142	AssertRCReturn(rc, rc);
2143
2144	pVmcsInfo->cEntryMsrLoad = cMsrs;
2145	pVmcsInfo->cExitMsrStore = cMsrs;
2146	pVmcsInfo->cExitMsrLoad = cMsrs;
2147	}
2148	return VINF_SUCCESS;
2149	}
2150
2151	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2152	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2153	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2154	}
2155
2156
2157	/**
2158	* Adds a new (or updates the value of an existing) guest/host MSR
2159	* pair to be swapped during the world-switch as part of the
2160	* auto-load/store MSR area in the VMCS.
2161	*
2162	* @returns VBox status code.
2163	* @param pVCpu The cross context virtual CPU structure.
2164	* @param pVmxTransient The VMX-transient structure.
2165	* @param idMsr The MSR.
2166	* @param uGuestMsrValue Value of the guest MSR.
2167	* @param fSetReadWrite Whether to set the guest read/write access of this
2168	* MSR (thus not causing a VM-exit).
2169	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2170	* necessary.
2171	*/
2172	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2173	bool fSetReadWrite, bool fUpdateHostMsr)
2174	{
2175	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2176	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2177	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2178	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2179	uint32_t i;
2180
2181	/* Paranoia. */
2182	Assert(pGuestMsrLoad);
2183
2184	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2185
2186	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2187	for (i = 0; i < cMsrs; i++)
2188	{
2189	if (pGuestMsrLoad[i].u32Msr == idMsr)
2190	break;
2191	}
2192
2193	bool fAdded = false;
2194	if (i == cMsrs)
2195	{
2196	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2197	++cMsrs;
2198	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2199	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2200
2201	/* Set the guest to read/write this MSR without causing VM-exits. */
2202	if ( fSetReadWrite
2203	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2204	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2205
2206	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
2207	fAdded = true;
2208	}
2209
2210	/* Update the MSR value for the newly added or already existing MSR. */
2211	pGuestMsrLoad[i].u32Msr = idMsr;
2212	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2213
2214	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2215	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2216	{
2217	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2218	pGuestMsrStore[i].u32Msr = idMsr;
2219	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2220	}
2221
2222	/* Update the corresponding slot in the host MSR area. */
2223	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2224	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2225	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2226	pHostMsr[i].u32Msr = idMsr;
2227
2228	/*
2229	* Only if the caller requests to update the host MSR value AND we've newly added the
2230	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2231	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2232	*
2233	* We do this for performance reasons since reading MSRs may be quite expensive.
2234	*/
2235	if (fAdded)
2236	{
2237	if (fUpdateHostMsr)
2238	{
2239	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2240	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2241	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2242	}
2243	else
2244	{
2245	/* Someone else can do the work. */
2246	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2247	}
2248	}
2249	return VINF_SUCCESS;
2250	}
2251
2252
2253	/**
2254	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2255	* auto-load/store MSR area in the VMCS.
2256	*
2257	* @returns VBox status code.
2258	* @param pVCpu The cross context virtual CPU structure.
2259	* @param pVmxTransient The VMX-transient structure.
2260	* @param idMsr The MSR.
2261	*/
2262	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2263	{
2264	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2265	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2266	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2267	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2268
2269	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2270
2271	for (uint32_t i = 0; i < cMsrs; i++)
2272	{
2273	/* Find the MSR. */
2274	if (pGuestMsrLoad[i].u32Msr == idMsr)
2275	{
2276	/*
2277	* If it's the last MSR, we only need to reduce the MSR count.
2278	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2279	*/
2280	if (i < cMsrs - 1)
2281	{
2282	/* Remove it from the VM-entry MSR-load area. */
2283	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2284	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2285
2286	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2287	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2288	{
2289	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2290	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2291	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2292	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2293	}
2294
2295	/* Remove it from the VM-exit MSR-load area. */
2296	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2297	Assert(pHostMsr[i].u32Msr == idMsr);
2298	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2299	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2300	}
2301
2302	/* Reduce the count to reflect the removed MSR and bail. */
2303	--cMsrs;
2304	break;
2305	}
2306	}
2307
2308	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2309	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2310	{
2311	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2312	AssertRCReturn(rc, rc);
2313
2314	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2315	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2316	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2317
2318	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2319	return VINF_SUCCESS;
2320	}
2321
2322	return VERR_NOT_FOUND;
2323	}
2324
2325
2326	/**
2327	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2328	*
2329	* @returns @c true if found, @c false otherwise.
2330	* @param pVmcsInfo The VMCS info. object.
2331	* @param idMsr The MSR to find.
2332	*/
2333	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2334	{
2335	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2336	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2337	Assert(pMsrs);
2338	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2339	for (uint32_t i = 0; i < cMsrs; i++)
2340	{
2341	if (pMsrs[i].u32Msr == idMsr)
2342	return true;
2343	}
2344	return false;
2345	}
2346
2347
2348	/**
2349	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2350	*
2351	* @param pVCpu The cross context virtual CPU structure.
2352	* @param pVmcsInfo The VMCS info. object.
2353	*
2354	* @remarks No-long-jump zone!!!
2355	*/
2356	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2357	{
2358	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2359
2360	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2361	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2362	Assert(pHostMsrLoad);
2363	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2364	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2365	for (uint32_t i = 0; i < cMsrs; i++)
2366	{
2367	/*
2368	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2369	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2370	*/
2371	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2372	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2373	else
2374	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2375	}
2376	}
2377
2378
2379	/**
2380	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2381	* perform lazy restoration of the host MSRs while leaving VT-x.
2382	*
2383	* @param pVCpu The cross context virtual CPU structure.
2384	*
2385	* @remarks No-long-jump zone!!!
2386	*/
2387	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2388	{
2389	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2390
2391	/*
2392	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2393	*/
2394	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2395	{
2396	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2397	#if HC_ARCH_BITS == 64
2398	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2399	{
2400	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2401	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2402	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2403	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2404	}
2405	#endif
2406	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2407	}
2408	}
2409
2410
2411	/**
2412	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2413	* lazily while leaving VT-x.
2414	*
2415	* @returns true if it does, false otherwise.
2416	* @param pVCpu The cross context virtual CPU structure.
2417	* @param idMsr The MSR to check.
2418	*/
2419	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2420	{
2421	NOREF(pVCpu);
2422	#if HC_ARCH_BITS == 64
2423	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2424	{
2425	switch (idMsr)
2426	{
2427	case MSR_K8_LSTAR:
2428	case MSR_K6_STAR:
2429	case MSR_K8_SF_MASK:
2430	case MSR_K8_KERNEL_GS_BASE:
2431	return true;
2432	}
2433	}
2434	#else
2435	RT_NOREF(pVCpu, idMsr);
2436	#endif
2437	return false;
2438	}
2439
2440
2441	/**
2442	* Loads a set of guests MSRs to allow read/passthru to the guest.
2443	*
2444	* The name of this function is slightly confusing. This function does NOT
2445	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2446	* common prefix for functions dealing with "lazy restoration" of the shared
2447	* MSRs.
2448	*
2449	* @param pVCpu The cross context virtual CPU structure.
2450	*
2451	* @remarks No-long-jump zone!!!
2452	*/
2453	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2454	{
2455	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2456	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2457
2458	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2459	#if HC_ARCH_BITS == 64
2460	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2461	{
2462	/*
2463	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2464	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2465	* we can skip a few MSR writes.
2466	*
2467	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2468	* guest MSR values in the guest-CPU context might be different to what's currently
2469	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2470	* CPU, see @bugref{8728}.
2471	*/
2472	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2473	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2474	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2475	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2476	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2477	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2478	{
2479	#ifdef VBOX_STRICT
2480	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2481	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2482	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2483	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2484	#endif
2485	}
2486	else
2487	{
2488	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2489	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2490	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2491	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2492	}
2493	}
2494	#endif
2495	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2496	}
2497
2498
2499	/**
2500	* Performs lazy restoration of the set of host MSRs if they were previously
2501	* loaded with guest MSR values.
2502	*
2503	* @param pVCpu The cross context virtual CPU structure.
2504	*
2505	* @remarks No-long-jump zone!!!
2506	* @remarks The guest MSRs should have been saved back into the guest-CPU
2507	* context by hmR0VmxImportGuestState()!!!
2508	*/
2509	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2510	{
2511	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2512	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2513
2514	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2515	{
2516	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2517	#if HC_ARCH_BITS == 64
2518	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2519	{
2520	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2521	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2522	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2523	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2524	}
2525	#endif
2526	}
2527	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2528	}
2529
2530
2531	/**
2532	* Verifies that our cached values of the VMCS fields are all consistent with
2533	* what's actually present in the VMCS.
2534	*
2535	* @returns VBox status code.
2536	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2537	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2538	* VMCS content. HMCPU error-field is
2539	* updated, see VMX_VCI_XXX.
2540	* @param pVCpu The cross context virtual CPU structure.
2541	* @param pVmcsInfo The VMCS info. object.
2542	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2543	*/
2544	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2545	{
2546	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2547
2548	uint32_t u32Val;
2549	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2550	AssertRCReturn(rc, rc);
2551	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2552	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2553	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2554	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2555
2556	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2557	AssertRCReturn(rc, rc);
2558	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2559	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2560	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2561	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2562
2563	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2564	AssertRCReturn(rc, rc);
2565	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2566	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2567	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2568	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2569
2570	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2571	AssertRCReturn(rc, rc);
2572	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2573	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2574	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2575	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2576
2577	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2578	{
2579	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2580	AssertRCReturn(rc, rc);
2581	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2582	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2583	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2584	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2585	}
2586
2587	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2588	AssertRCReturn(rc, rc);
2589	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2590	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2591	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2592	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2593
2594	uint64_t u64Val;
2595	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2596	AssertRCReturn(rc, rc);
2597	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2598	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2599	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2600	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2601
2602	NOREF(pcszVmcs);
2603	return VINF_SUCCESS;
2604	}
2605
2606
2607	#ifdef VBOX_STRICT
2608	/**
2609	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2610	*
2611	* @param pVCpu The cross context virtual CPU structure.
2612	* @param pVmcsInfo The VMCS info. object.
2613	*/
2614	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2615	{
2616	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2617
2618	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2619	{
2620	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2621	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2622	uint64_t uVmcsEferMsrVmcs;
2623	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2624	AssertRC(rc);
2625
2626	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2627	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2628	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2629	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2630	}
2631	}
2632
2633
2634	/**
2635	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2636	* VMCS are correct.
2637	*
2638	* @param pVCpu The cross context virtual CPU structure.
2639	* @param pVmcsInfo The VMCS info. object.
2640	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2641	*/
2642	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2643	{
2644	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2645
2646	/* Read the various MSR-area counts from the VMCS. */
2647	uint32_t cEntryLoadMsrs;
2648	uint32_t cExitStoreMsrs;
2649	uint32_t cExitLoadMsrs;
2650	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2651	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2652	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2653
2654	/* Verify all the MSR counts are the same. */
2655	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2656	Assert(cExitStoreMsrs == cExitLoadMsrs);
2657	uint32_t const cMsrs = cExitLoadMsrs;
2658
2659	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2660	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2661
2662	/* Verify the MSR counts are within the allocated page size. */
2663	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2664
2665	/* Verify the relevant contents of the MSR areas match. */
2666	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2667	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2668	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2669	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2670	for (uint32_t i = 0; i < cMsrs; i++)
2671	{
2672	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2673	if (fSeparateExitMsrStorePage)
2674	{
2675	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2676	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2677	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2678	}
2679
2680	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2681	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2682	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2683
2684	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2685	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2686	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2687	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2688
2689	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2690	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2691	if (fIsEferMsr)
2692	{
2693	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2694	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2695	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2696	}
2697
2698	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2699	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2700	{
2701	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2702	if (fIsEferMsr)
2703	{
2704	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2705	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2706	}
2707	else
2708	{
2709	if (!fIsNstGstVmcs)
2710	{
2711	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2712	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2713	}
2714	else
2715	{
2716	/*
2717	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2718	* execute a nested-guest with MSR passthrough.
2719	*
2720	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2721	* allow passthrough too.
2722	*/
2723	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2724	Assert(pvMsrBitmapNstGst);
2725	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2726	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2727	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2728	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2729	}
2730	}
2731	}
2732
2733	/* Move to the next MSR. */
2734	pHostMsrLoad++;
2735	pGuestMsrLoad++;
2736	pGuestMsrStore++;
2737	}
2738	}
2739	#endif /* VBOX_STRICT */
2740
2741
2742	/**
2743	* Flushes the TLB using EPT.
2744	*
2745	* @returns VBox status code.
2746	* @param pVCpu The cross context virtual CPU structure of the calling
2747	* EMT. Can be NULL depending on @a enmTlbFlush.
2748	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2749	* enmTlbFlush.
2750	* @param enmTlbFlush Type of flush.
2751	*
2752	* @remarks Caller is responsible for making sure this function is called only
2753	* when NestedPaging is supported and providing @a enmTlbFlush that is
2754	* supported by the CPU.
2755	* @remarks Can be called with interrupts disabled.
2756	*/
2757	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2758	{
2759	uint64_t au64Descriptor[2];
2760	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2761	au64Descriptor[0] = 0;
2762	else
2763	{
2764	Assert(pVCpu);
2765	Assert(pVmcsInfo);
2766	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2767	}
2768	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2769
2770	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2771	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2772
2773	if ( RT_SUCCESS(rc)
2774	&& pVCpu)
2775	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2776	}
2777
2778
2779	/**
2780	* Flushes the TLB using VPID.
2781	*
2782	* @returns VBox status code.
2783	* @param pVCpu The cross context virtual CPU structure of the calling
2784	* EMT. Can be NULL depending on @a enmTlbFlush.
2785	* @param enmTlbFlush Type of flush.
2786	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2787	* on @a enmTlbFlush).
2788	*
2789	* @remarks Can be called with interrupts disabled.
2790	*/
2791	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2792	{
2793	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2794
2795	uint64_t au64Descriptor[2];
2796	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2797	{
2798	au64Descriptor[0] = 0;
2799	au64Descriptor[1] = 0;
2800	}
2801	else
2802	{
2803	AssertPtr(pVCpu);
2804	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2805	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2806	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2807	au64Descriptor[1] = GCPtr;
2808	}
2809
2810	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2811	AssertMsg(rc == VINF_SUCCESS,
2812	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2813
2814	if ( RT_SUCCESS(rc)
2815	&& pVCpu)
2816	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2817	NOREF(rc);
2818	}
2819
2820
2821	/**
2822	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2823	* otherwise there is nothing really to invalidate.
2824	*
2825	* @returns VBox status code.
2826	* @param pVCpu The cross context virtual CPU structure.
2827	* @param GCVirt Guest virtual address of the page to invalidate.
2828	*/
2829	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2830	{
2831	AssertPtr(pVCpu);
2832	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2833
2834	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2835	{
2836	/*
2837	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2838	* the EPT case. See @bugref{6043} and @bugref{6177}.
2839	*
2840	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2841	* as this function maybe called in a loop with individual addresses.
2842	*/
2843	PVM pVM = pVCpu->CTX_SUFF(pVM);
2844	if (pVM->hm.s.vmx.fVpid)
2845	{
2846	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2847
2848	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2849	/*
2850	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2851	* where executing INVVPID outside 64-bit mode does not flush translations of
2852	* 64-bit linear addresses, see @bugref{6208#c72}.
2853	*/
2854	if (RT_HI_U32(GCVirt))
2855	fVpidFlush = false;
2856	#endif
2857
2858	if (fVpidFlush)
2859	{
2860	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2861	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2862	}
2863	else
2864	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2865	}
2866	else if (pVM->hm.s.fNestedPaging)
2867	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2868	}
2869
2870	return VINF_SUCCESS;
2871	}
2872
2873
2874	/**
2875	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2876	* case where neither EPT nor VPID is supported by the CPU.
2877	*
2878	* @param pHostCpu The HM physical-CPU structure.
2879	* @param pVCpu The cross context virtual CPU structure.
2880	*
2881	* @remarks Called with interrupts disabled.
2882	*/
2883	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2884	{
2885	AssertPtr(pVCpu);
2886	AssertPtr(pHostCpu);
2887
2888	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2889
2890	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2891	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2892	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2893	pVCpu->hm.s.fForceTLBFlush = false;
2894	return;
2895	}
2896
2897
2898	/**
2899	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2900	*
2901	* @param pHostCpu The HM physical-CPU structure.
2902	* @param pVCpu The cross context virtual CPU structure.
2903	* @param pVmcsInfo The VMCS info. object.
2904	*
2905	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2906	* nomenclature. The reason is, to avoid confusion in compare statements
2907	* since the host-CPU copies are named "ASID".
2908	*
2909	* @remarks Called with interrupts disabled.
2910	*/
2911	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2912	{
2913	#ifdef VBOX_WITH_STATISTICS
2914	bool fTlbFlushed = false;
2915	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2916	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2917	if (!fTlbFlushed) \
2918	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2919	} while (0)
2920	#else
2921	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2922	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2923	#endif
2924
2925	AssertPtr(pVCpu);
2926	AssertPtr(pHostCpu);
2927	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2928
2929	PVM pVM = pVCpu->CTX_SUFF(pVM);
2930	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2931	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2932	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2933
2934	/*
2935	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2936	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2937	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2938	* cannot reuse the current ASID anymore.
2939	*/
2940	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2941	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2942	{
2943	++pHostCpu->uCurrentAsid;
2944	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2945	{
2946	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2947	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2948	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2949	}
2950
2951	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2952	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2953	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2954
2955	/*
2956	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2957	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2958	*/
2959	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2960	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2961	HMVMX_SET_TAGGED_TLB_FLUSHED();
2962	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2963	}
2964	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2965	{
2966	/*
2967	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2968	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2969	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2970	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2971	* mappings, see @bugref{6568}.
2972	*
2973	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2974	*/
2975	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2976	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2977	HMVMX_SET_TAGGED_TLB_FLUSHED();
2978	}
2979
2980	pVCpu->hm.s.fForceTLBFlush = false;
2981	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2982
2983	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2984	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2985	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2986	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2987	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2988	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2989	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2990	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2991	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2992
2993	/* Update VMCS with the VPID. */
2994	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2995	AssertRC(rc);
2996
2997	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2998	}
2999
3000
3001	/**
3002	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3003	*
3004	* @param pHostCpu The HM physical-CPU structure.
3005	* @param pVCpu The cross context virtual CPU structure.
3006	* @param pVmcsInfo The VMCS info. object.
3007	*
3008	* @remarks Called with interrupts disabled.
3009	*/
3010	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3011	{
3012	AssertPtr(pVCpu);
3013	AssertPtr(pHostCpu);
3014	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3015	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3016	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3017
3018	/*
3019	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3020	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3021	*/
3022	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3023	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3024	{
3025	pVCpu->hm.s.fForceTLBFlush = true;
3026	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3027	}
3028
3029	/* Check for explicit TLB flushes. */
3030	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3031	{
3032	pVCpu->hm.s.fForceTLBFlush = true;
3033	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3034	}
3035
3036	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3037	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3038
3039	if (pVCpu->hm.s.fForceTLBFlush)
3040	{
3041	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3042	pVCpu->hm.s.fForceTLBFlush = false;
3043	}
3044	}
3045
3046
3047	/**
3048	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3049	*
3050	* @param pHostCpu The HM physical-CPU structure.
3051	* @param pVCpu The cross context virtual CPU structure.
3052	*
3053	* @remarks Called with interrupts disabled.
3054	*/
3055	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3056	{
3057	AssertPtr(pVCpu);
3058	AssertPtr(pHostCpu);
3059	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3060	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3061	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3062
3063	/*
3064	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3065	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3066	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3067	* cannot reuse the current ASID anymore.
3068	*/
3069	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3070	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3071	{
3072	pVCpu->hm.s.fForceTLBFlush = true;
3073	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3074	}
3075
3076	/* Check for explicit TLB flushes. */
3077	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3078	{
3079	/*
3080	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3081	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3082	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3083	* include fExplicitFlush's too) - an obscure corner case.
3084	*/
3085	pVCpu->hm.s.fForceTLBFlush = true;
3086	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3087	}
3088
3089	PVM pVM = pVCpu->CTX_SUFF(pVM);
3090	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3091	if (pVCpu->hm.s.fForceTLBFlush)
3092	{
3093	++pHostCpu->uCurrentAsid;
3094	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3095	{
3096	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3097	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3098	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3099	}
3100
3101	pVCpu->hm.s.fForceTLBFlush = false;
3102	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3103	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3104	if (pHostCpu->fFlushAsidBeforeUse)
3105	{
3106	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3107	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3108	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3109	{
3110	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3111	pHostCpu->fFlushAsidBeforeUse = false;
3112	}
3113	else
3114	{
3115	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3116	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3117	}
3118	}
3119	}
3120
3121	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3122	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3123	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3124	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3125	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3126	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3127	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3128
3129	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3130	AssertRC(rc);
3131	}
3132
3133
3134	/**
3135	* Flushes the guest TLB entry based on CPU capabilities.
3136	*
3137	* @param pHostCpu The HM physical-CPU structure.
3138	* @param pVCpu The cross context virtual CPU structure.
3139	* @param pVmcsInfo The VMCS info. object.
3140	*
3141	* @remarks Called with interrupts disabled.
3142	*/
3143	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3144	{
3145	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3146	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3147	#endif
3148	PVM pVM = pVCpu->CTX_SUFF(pVM);
3149	switch (pVM->hm.s.vmx.enmTlbFlushType)
3150	{
3151	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3152	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3153	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3154	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3155	default:
3156	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3157	break;
3158	}
3159	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3160	}
3161
3162
3163	/**
3164	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3165	* TLB entries from the host TLB before VM-entry.
3166	*
3167	* @returns VBox status code.
3168	* @param pVM The cross context VM structure.
3169	*/
3170	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3171	{
3172	/*
3173	* Determine optimal flush type for nested paging.
3174	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3175	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3176	*/
3177	if (pVM->hm.s.fNestedPaging)
3178	{
3179	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3180	{
3181	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3182	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3183	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3184	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3185	else
3186	{
3187	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3188	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3189	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3190	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3191	}
3192
3193	/* Make sure the write-back cacheable memory type for EPT is supported. */
3194	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3195	{
3196	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3197	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3198	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3199	}
3200
3201	/* EPT requires a page-walk length of 4. */
3202	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3203	{
3204	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3205	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3206	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3207	}
3208	}
3209	else
3210	{
3211	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3212	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3213	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3214	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3215	}
3216	}
3217
3218	/*
3219	* Determine optimal flush type for VPID.
3220	*/
3221	if (pVM->hm.s.vmx.fVpid)
3222	{
3223	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3224	{
3225	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3226	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3227	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3228	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3229	else
3230	{
3231	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3232	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3233	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3234	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3235	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3236	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3237	pVM->hm.s.vmx.fVpid = false;
3238	}
3239	}
3240	else
3241	{
3242	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3243	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3244	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3245	pVM->hm.s.vmx.fVpid = false;
3246	}
3247	}
3248
3249	/*
3250	* Setup the handler for flushing tagged-TLBs.
3251	*/
3252	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3253	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3254	else if (pVM->hm.s.fNestedPaging)
3255	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3256	else if (pVM->hm.s.vmx.fVpid)
3257	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3258	else
3259	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3260	return VINF_SUCCESS;
3261	}
3262
3263
3264	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3265	/**
3266	* Sets up the shadow VMCS fields arrays.
3267	*
3268	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3269	* executing the guest.
3270	*
3271	* @returns VBox status code.
3272	* @param pVM The cross context VM structure.
3273	*/
3274	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3275	{
3276	/*
3277	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3278	* when the host does not support it.
3279	*/
3280	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3281	if ( !fGstVmwriteAll
3282	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3283	{ /* likely. */ }
3284	else
3285	{
3286	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3287	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3288	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3289	}
3290
3291	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3292	uint32_t cRwFields = 0;
3293	uint32_t cRoFields = 0;
3294	for (uint32_t i = 0; i < cVmcsFields; i++)
3295	{
3296	VMXVMCSFIELD VmcsField;
3297	VmcsField.u = g_aVmcsFields[i];
3298
3299	/*
3300	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3301	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3302	* in the shadow VMCS fields array as they would be redundant.
3303	*
3304	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3305	* we must not include it in the shadow VMCS fields array. Guests attempting to
3306	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3307	* the required behavior.
3308	*/
3309	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3310	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3311	{
3312	/*
3313	* Read-only fields are placed in a separate array so that while syncing shadow
3314	* VMCS fields later (which is more performance critical) we can avoid branches.
3315	*
3316	* However, if the guest can write to all fields (including read-only fields),
3317	* we treat it a as read/write field. Otherwise, writing to these fields would
3318	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3319	*/
3320	if ( fGstVmwriteAll
3321	\|\| !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3322	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3323	else
3324	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3325	}
3326	}
3327
3328	/* Update the counts. */
3329	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3330	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3331	return VINF_SUCCESS;
3332	}
3333
3334
3335	/**
3336	* Sets up the VMREAD and VMWRITE bitmaps.
3337	*
3338	* @param pVM The cross context VM structure.
3339	*/
3340	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3341	{
3342	/*
3343	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3344	*/
3345	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3346	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3347	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3348	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3349	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3350
3351	/*
3352	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3353	* VMREAD and VMWRITE bitmaps.
3354	*/
3355	{
3356	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3357	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3358	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3359	{
3360	uint32_t const uVmcsField = paShadowVmcsFields[i];
3361	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3362	Assert(uVmcsField >> 3 < cbBitmap);
3363	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3364	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3365	}
3366	}
3367
3368	/*
3369	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3370	* if the host supports VMWRITE to all supported VMCS fields.
3371	*/
3372	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3373	{
3374	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3375	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3376	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3377	{
3378	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3379	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3380	Assert(uVmcsField >> 3 < cbBitmap);
3381	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3382	}
3383	}
3384	}
3385	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3386
3387
3388	/**
3389	* Sets up the virtual-APIC page address for the VMCS.
3390	*
3391	* @returns VBox status code.
3392	* @param pVCpu The cross context virtual CPU structure.
3393	* @param pVmcsInfo The VMCS info. object.
3394	*/
3395	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3396	{
3397	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3398	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3399	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3400	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3401	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3402	}
3403
3404
3405	/**
3406	* Sets up the MSR-bitmap address for the VMCS.
3407	*
3408	* @returns VBox status code.
3409	* @param pVCpu The cross context virtual CPU structure.
3410	* @param pVmcsInfo The VMCS info. object.
3411	*/
3412	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3413	{
3414	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3415	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3416	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3417	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3418	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3419	}
3420
3421
3422	/**
3423	* Sets up the APIC-access page address for the VMCS.
3424	*
3425	* @returns VBox status code.
3426	* @param pVCpu The cross context virtual CPU structure.
3427	*/
3428	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3429	{
3430	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3431	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3432	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3433	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3434	}
3435
3436
3437	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3438	/**
3439	* Sets up the VMREAD bitmap address for the VMCS.
3440	*
3441	* @returns VBox status code.
3442	* @param pVCpu The cross context virtual CPU structure.
3443	*/
3444	DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3445	{
3446	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3447	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3448	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3449	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3450	}
3451
3452
3453	/**
3454	* Sets up the VMWRITE bitmap address for the VMCS.
3455	*
3456	* @returns VBox status code.
3457	* @param pVCpu The cross context virtual CPU structure.
3458	*/
3459	DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3460	{
3461	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3462	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3463	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3464	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3465	}
3466	#endif
3467
3468
3469	/**
3470	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3471	* in the VMCS.
3472	*
3473	* @returns VBox status code.
3474	* @param pVCpu The cross context virtual CPU structure.
3475	* @param pVmcsInfo The VMCS info. object.
3476	*/
3477	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3478	{
3479	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3480
3481	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3482	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3483	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3484
3485	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3486	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3487	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3488
3489	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3490	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3491	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3492
3493	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3494	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3495	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3496	AssertRCReturn(rc, rc);
3497	return VINF_SUCCESS;
3498	}
3499
3500
3501	/**
3502	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3503	*
3504	* @param pVCpu The cross context virtual CPU structure.
3505	* @param pVmcsInfo The VMCS info. object.
3506	*/
3507	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3508	{
3509	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3510
3511	/*
3512	* The guest can access the following MSRs (read, write) without causing
3513	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3514	*/
3515	PVM pVM = pVCpu->CTX_SUFF(pVM);
3516	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3517	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3518	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3519	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3520	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3521
3522	/*
3523	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3524	* associated with then. We never need to intercept access (writes need to be
3525	* executed without causing a VM-exit, reads will #GP fault anyway).
3526	*
3527	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3528	* read/write them. We swap the the guest/host MSR value using the
3529	* auto-load/store MSR area.
3530	*/
3531	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3532	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3533	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3534	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3535	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3536	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3537
3538	#if HC_ARCH_BITS == 64
3539	/*
3540	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3541	* required for 64-bit guests.
3542	*/
3543	if (pVM->hm.s.fAllow64BitGuests)
3544	{
3545	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3546	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3547	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3548	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3549	}
3550	#endif
3551
3552	/*
3553	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3554	*/
3555	#ifdef VBOX_STRICT
3556	Assert(pVmcsInfo->pvMsrBitmap);
3557	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3558	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3559	#endif
3560	}
3561
3562
3563	/**
3564	* Sets up pin-based VM-execution controls in the VMCS.
3565	*
3566	* @returns VBox status code.
3567	* @param pVCpu The cross context virtual CPU structure.
3568	* @param pVmcsInfo The VMCS info. object.
3569	*/
3570	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3571	{
3572	PVM pVM = pVCpu->CTX_SUFF(pVM);
3573	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3574	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3575
3576	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3577	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3578
3579	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3580	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3581
3582	/* Enable the VMX-preemption timer. */
3583	if (pVM->hm.s.vmx.fUsePreemptTimer)
3584	{
3585	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3586	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3587	}
3588
3589	#if 0
3590	/* Enable posted-interrupt processing. */
3591	if (pVM->hm.s.fPostedIntrs)
3592	{
3593	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3594	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3595	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3596	}
3597	#endif
3598
3599	if ((fVal & fZap) != fVal)
3600	{
3601	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3602	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3603	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3604	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3605	}
3606
3607	/* Commit it to the VMCS and update our cache. */
3608	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3609	AssertRCReturn(rc, rc);
3610	pVmcsInfo->u32PinCtls = fVal;
3611
3612	return VINF_SUCCESS;
3613	}
3614
3615
3616	/**
3617	* Sets up secondary processor-based VM-execution controls in the VMCS.
3618	*
3619	* @returns VBox status code.
3620	* @param pVCpu The cross context virtual CPU structure.
3621	* @param pVmcsInfo The VMCS info. object.
3622	*/
3623	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3624	{
3625	PVM pVM = pVCpu->CTX_SUFF(pVM);
3626	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3627	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3628
3629	/* WBINVD causes a VM-exit. */
3630	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3631	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3632
3633	/* Enable EPT (aka nested-paging). */
3634	if (pVM->hm.s.fNestedPaging)
3635	fVal \|= VMX_PROC_CTLS2_EPT;
3636
3637	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3638	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3639	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3640	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3641	fVal \|= VMX_PROC_CTLS2_INVPCID;
3642
3643	/* Enable VPID. */
3644	if (pVM->hm.s.vmx.fVpid)
3645	fVal \|= VMX_PROC_CTLS2_VPID;
3646
3647	/* Enable unrestricted guest execution. */
3648	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3649	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3650
3651	#if 0
3652	if (pVM->hm.s.fVirtApicRegs)
3653	{
3654	/* Enable APIC-register virtualization. */
3655	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3656	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3657
3658	/* Enable virtual-interrupt delivery. */
3659	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3660	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3661	}
3662	#endif
3663
3664	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3665	where the TPR shadow resides. */
3666	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3667	* done dynamically. */
3668	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3669	{
3670	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3671	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3672	AssertRCReturn(rc, rc);
3673	}
3674
3675	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3676	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3677	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3678	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3679	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3680
3681	/* Enable Pause-Loop exiting. */
3682	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3683	&& pVM->hm.s.vmx.cPleGapTicks
3684	&& pVM->hm.s.vmx.cPleWindowTicks)
3685	{
3686	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3687
3688	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3689	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3690	AssertRCReturn(rc, rc);
3691	}
3692
3693	if ((fVal & fZap) != fVal)
3694	{
3695	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3696	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3697	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3698	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3699	}
3700
3701	/* Commit it to the VMCS and update our cache. */
3702	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3703	AssertRCReturn(rc, rc);
3704	pVmcsInfo->u32ProcCtls2 = fVal;
3705
3706	return VINF_SUCCESS;
3707	}
3708
3709
3710	/**
3711	* Sets up processor-based VM-execution controls in the VMCS.
3712	*
3713	* @returns VBox status code.
3714	* @param pVCpu The cross context virtual CPU structure.
3715	* @param pVmcsInfo The VMCS info. object.
3716	*/
3717	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3718	{
3719	PVM pVM = pVCpu->CTX_SUFF(pVM);
3720
3721	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3722	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3723
3724	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3725	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3726	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3727	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3728	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3729	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3730	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3731
3732	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3733	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3734	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3735	{
3736	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3737	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3738	}
3739
3740	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3741	if (!pVM->hm.s.fNestedPaging)
3742	{
3743	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3744	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3745	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3746	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3747	}
3748
3749	/* Use TPR shadowing if supported by the CPU. */
3750	if ( PDMHasApic(pVM)
3751	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3752	{
3753	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3754	/* CR8 writes cause a VM-exit based on TPR threshold. */
3755	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3756	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3757	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3758	AssertRCReturn(rc, rc);
3759	}
3760	else
3761	{
3762	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3763	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3764	if (pVM->hm.s.fAllow64BitGuests)
3765	{
3766	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3767	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3768	}
3769	}
3770
3771	/* Use MSR-bitmaps if supported by the CPU. */
3772	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3773	{
3774	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3775	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3776	AssertRCReturn(rc, rc);
3777	}
3778
3779	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3780	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3781	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3782
3783	if ((fVal & fZap) != fVal)
3784	{
3785	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3786	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3787	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3788	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3789	}
3790
3791	/* Commit it to the VMCS and update our cache. */
3792	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3793	AssertRCReturn(rc, rc);
3794	pVmcsInfo->u32ProcCtls = fVal;
3795
3796	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3797	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3798	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3799
3800	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3801	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3802	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3803
3804	/* Sanity check, should not really happen. */
3805	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3806	{ /* likely */ }
3807	else
3808	{
3809	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3810	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3811	}
3812
3813	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3814	return VINF_SUCCESS;
3815	}
3816
3817
3818	/**
3819	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3820	* Processor-based VM-execution) control fields in the VMCS.
3821	*
3822	* @returns VBox status code.
3823	* @param pVCpu The cross context virtual CPU structure.
3824	* @param pVmcsInfo The VMCS info. object.
3825	*/
3826	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3827	{
3828	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3829	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3830	{
3831	int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3832	rc \|= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3833	if (RT_SUCCESS(rc))
3834	{ /* likely */ }
3835	else
3836	{
3837	LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3838	return rc;
3839	}
3840	}
3841	#endif
3842
3843	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3844	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3845	if (RT_SUCCESS(rc))
3846	{
3847	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3848	if (RT_SUCCESS(rc))
3849	{
3850	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3851	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3852	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3853	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3854	if (RT_SUCCESS(rc))
3855	{
3856	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3857	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3858	return VINF_SUCCESS;
3859	}
3860	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3861	}
3862	else
3863	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3864	}
3865	else
3866	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3867	return rc;
3868	}
3869
3870
3871	/**
3872	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3873	*
3874	* We shall setup those exception intercepts that don't change during the
3875	* lifetime of the VM here. The rest are done dynamically while loading the
3876	* guest state.
3877	*
3878	* @returns VBox status code.
3879	* @param pVCpu The cross context virtual CPU structure.
3880	* @param pVmcsInfo The VMCS info. object.
3881	*/
3882	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3883	{
3884	/*
3885	* The following exceptions are always intercepted:
3886	*
3887	* #AC - To prevent the guest from hanging the CPU.
3888	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3889	* recursive #DBs can cause a CPU hang.
3890	* #PF - To sync our shadow page tables when nested-paging is not used.
3891	*/
3892	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3893	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3894	\| RT_BIT(X86_XCPT_DB)
3895	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3896
3897	/* Commit it to the VMCS. */
3898	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3899	AssertRCReturn(rc, rc);
3900
3901	/* Update our cache of the exception bitmap. */
3902	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3903	return VINF_SUCCESS;
3904	}
3905
3906
3907	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3908	/**
3909	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3910	*
3911	* @returns VBox status code.
3912	* @param pVCpu The cross context virtual CPU structure.
3913	* @param pVmcsInfo The VMCS info. object.
3914	*/
3915	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3916	{
3917	PVM pVM = pVCpu->CTX_SUFF(pVM);
3918	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3919	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3920	if (RT_SUCCESS(rc))
3921	{
3922	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3923	if (RT_SUCCESS(rc))
3924	{
3925	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3926	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3927	if (RT_SUCCESS(rc))
3928	{
3929	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3930	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3931	if (RT_SUCCESS(rc))
3932	return VINF_SUCCESS;
3933
3934	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3935	}
3936	else
3937	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3938	}
3939	else
3940	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3941	}
3942	else
3943	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3944
3945	return rc;
3946	}
3947	#endif
3948
3949
3950	/**
3951	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3952	* VMX.
3953	*
3954	* @returns VBox status code.
3955	* @param pVCpu The cross context virtual CPU structure.
3956	* @param pVmcsInfo The VMCS info. object.
3957	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3958	*/
3959	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3960	{
3961	Assert(pVmcsInfo->pvVmcs);
3962	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3963
3964	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3965	PVM pVM = pVCpu->CTX_SUFF(pVM);
3966	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3967	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3968
3969	LogFlowFunc(("\n"));
3970
3971	/*
3972	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3973	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3974	*/
3975	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3976	if (RT_SUCCESS(rc))
3977	{
3978	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3979	if (RT_SUCCESS(rc))
3980	{
3981	if (!fIsNstGstVmcs)
3982	{
3983	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3984	if (RT_SUCCESS(rc))
3985	{
3986	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3987	if (RT_SUCCESS(rc))
3988	{
3989	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3990	if (RT_SUCCESS(rc))
3991	{
3992	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3993	if (RT_SUCCESS(rc))
3994	{
3995	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3996	/*
3997	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
3998	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
3999	* making it fit for use when VMCS shadowing is later enabled.
4000	*/
4001	if (pVmcsInfo->pvShadowVmcs)
4002	{
4003	VMXVMCSREVID VmcsRevId;
4004	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4005	VmcsRevId.n.fIsShadowVmcs = 1;
4006	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4007	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4008	if (RT_SUCCESS(rc))
4009	{ /* likely */ }
4010	else
4011	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4012	}
4013	#endif
4014	}
4015	else
4016	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4017	}
4018	else
4019	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4020	}
4021	else
4022	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4023	}
4024	else
4025	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4026	}
4027	else
4028	{
4029	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4030	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4031	if (RT_SUCCESS(rc))
4032	{ /* likely */ }
4033	else
4034	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4035	#else
4036	AssertFailed();
4037	#endif
4038	}
4039	}
4040	else
4041	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4042	}
4043	else
4044	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4045
4046	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4047	if (RT_SUCCESS(rc))
4048	{
4049	rc = hmR0VmxClearVmcs(pVmcsInfo);
4050	if (RT_SUCCESS(rc))
4051	{ /* likely */ }
4052	else
4053	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4054	}
4055
4056	/*
4057	* Update the last-error record both for failures and success, so we
4058	* can propagate the status code back to ring-3 for diagnostics.
4059	*/
4060	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4061	NOREF(pszVmcs);
4062	return rc;
4063	}
4064
4065
4066	/**
4067	* Does global VT-x initialization (called during module initialization).
4068	*
4069	* @returns VBox status code.
4070	*/
4071	VMMR0DECL(int) VMXR0GlobalInit(void)
4072	{
4073	#ifdef HMVMX_USE_FUNCTION_TABLE
4074	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4075	# ifdef VBOX_STRICT
4076	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4077	Assert(g_apfnVMExitHandlers[i]);
4078	# endif
4079	#endif
4080	return VINF_SUCCESS;
4081	}
4082
4083
4084	/**
4085	* Does global VT-x termination (called during module termination).
4086	*/
4087	VMMR0DECL(void) VMXR0GlobalTerm()
4088	{
4089	/* Nothing to do currently. */
4090	}
4091
4092
4093	/**
4094	* Sets up and activates VT-x on the current CPU.
4095	*
4096	* @returns VBox status code.
4097	* @param pHostCpu The HM physical-CPU structure.
4098	* @param pVM The cross context VM structure. Can be
4099	* NULL after a host resume operation.
4100	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4101	* fEnabledByHost is @c true).
4102	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4103	* @a fEnabledByHost is @c true).
4104	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4105	* enable VT-x on the host.
4106	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4107	*/
4108	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4109	PCSUPHWVIRTMSRS pHwvirtMsrs)
4110	{
4111	AssertPtr(pHostCpu);
4112	AssertPtr(pHwvirtMsrs);
4113	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4114
4115	/* Enable VT-x if it's not already enabled by the host. */
4116	if (!fEnabledByHost)
4117	{
4118	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4119	if (RT_FAILURE(rc))
4120	return rc;
4121	}
4122
4123	/*
4124	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4125	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4126	* invalidated when flushing by VPID.
4127	*/
4128	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4129	{
4130	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4131	pHostCpu->fFlushAsidBeforeUse = false;
4132	}
4133	else
4134	pHostCpu->fFlushAsidBeforeUse = true;
4135
4136	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4137	++pHostCpu->cTlbFlushes;
4138
4139	return VINF_SUCCESS;
4140	}
4141
4142
4143	/**
4144	* Deactivates VT-x on the current CPU.
4145	*
4146	* @returns VBox status code.
4147	* @param pvCpuPage Pointer to the VMXON region.
4148	* @param HCPhysCpuPage Physical address of the VMXON region.
4149	*
4150	* @remarks This function should never be called when SUPR0EnableVTx() or
4151	* similar was used to enable VT-x on the host.
4152	*/
4153	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4154	{
4155	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4156
4157	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4158	return hmR0VmxLeaveRootMode();
4159	}
4160
4161
4162	/**
4163	* Does per-VM VT-x initialization.
4164	*
4165	* @returns VBox status code.
4166	* @param pVM The cross context VM structure.
4167	*/
4168	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4169	{
4170	AssertPtr(pVM);
4171	LogFlowFunc(("pVM=%p\n", pVM));
4172
4173	int rc = hmR0VmxStructsAlloc(pVM);
4174	if (RT_FAILURE(rc))
4175	{
4176	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4177	return rc;
4178	}
4179
4180	return VINF_SUCCESS;
4181	}
4182
4183
4184	/**
4185	* Does per-VM VT-x termination.
4186	*
4187	* @returns VBox status code.
4188	* @param pVM The cross context VM structure.
4189	*/
4190	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4191	{
4192	AssertPtr(pVM);
4193	LogFlowFunc(("pVM=%p\n", pVM));
4194
4195	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4196	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4197	{
4198	Assert(pVM->hm.s.vmx.pvScratch);
4199	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4200	}
4201	#endif
4202	hmR0VmxStructsFree(pVM);
4203	return VINF_SUCCESS;
4204	}
4205
4206
4207	/**
4208	* Sets up the VM for execution using hardware-assisted VMX.
4209	* This function is only called once per-VM during initialization.
4210	*
4211	* @returns VBox status code.
4212	* @param pVM The cross context VM structure.
4213	*/
4214	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4215	{
4216	AssertPtr(pVM);
4217	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4218
4219	LogFlowFunc(("pVM=%p\n", pVM));
4220
4221	/*
4222	* At least verify if VMX is enabled, since we can't check if we're in
4223	* VMX root mode or not without causing a #GP.
4224	*/
4225	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4226	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4227	{ /* likely */ }
4228	else
4229	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4230
4231	/*
4232	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4233	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4234	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4235	*/
4236	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4237	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4238	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4239	{
4240	LogRelFunc(("Invalid real-on-v86 state.\n"));
4241	return VERR_INTERNAL_ERROR;
4242	}
4243
4244	/* Initialize these always, see hmR3InitFinalizeR0().*/
4245	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4246	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4247
4248	/* Setup the tagged-TLB flush handlers. */
4249	int rc = hmR0VmxSetupTaggedTlb(pVM);
4250	if (RT_FAILURE(rc))
4251	{
4252	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4253	return rc;
4254	}
4255
4256	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4257	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4258	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4259	{
4260	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4261	if (RT_SUCCESS(rc))
4262	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4263	else
4264	{
4265	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4266	return rc;
4267	}
4268	}
4269	#endif
4270
4271	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4272	{
4273	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4274	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4275
4276	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4277	if (RT_SUCCESS(rc))
4278	{
4279	#if HC_ARCH_BITS == 32
4280	hmR0VmxInitVmcsReadCache(pVCpu);
4281	#endif
4282	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4283	if (pVM->cpum.ro.GuestFeatures.fVmx)
4284	{
4285	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4286	if (RT_SUCCESS(rc))
4287	{ /* likely */ }
4288	else
4289	{
4290	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4291	return rc;
4292	}
4293	}
4294	#endif
4295	}
4296	else
4297	{
4298	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4299	return rc;
4300	}
4301	}
4302
4303	return VINF_SUCCESS;
4304	}
4305
4306
4307	#if HC_ARCH_BITS == 32
4308	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4309	/**
4310	* Check if guest state allows safe use of 32-bit switcher again.
4311	*
4312	* Segment bases and protected mode structures must be 32-bit addressable
4313	* because the 32-bit switcher will ignore high dword when writing these VMCS
4314	* fields. See @bugref{8432} for details.
4315	*
4316	* @returns true if safe, false if must continue to use the 64-bit switcher.
4317	* @param pCtx Pointer to the guest-CPU context.
4318	*
4319	* @remarks No-long-jump zone!!!
4320	*/
4321	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4322	{
4323	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4324	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4325	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4326	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4327	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4328	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4329	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4330	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4331	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4332	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4333
4334	/* All good, bases are 32-bit. */
4335	return true;
4336	}
4337	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
4338
4339	# ifdef VBOX_STRICT
4340	static bool hmR0VmxIsValidWriteFieldInCache(uint32_t idxField)
4341	{
4342	switch (idxField)
4343	{
4344	case VMX_VMCS_GUEST_RIP:
4345	case VMX_VMCS_GUEST_RSP:
4346	case VMX_VMCS_GUEST_SYSENTER_EIP:
4347	case VMX_VMCS_GUEST_SYSENTER_ESP:
4348	case VMX_VMCS_GUEST_GDTR_BASE:
4349	case VMX_VMCS_GUEST_IDTR_BASE:
4350	case VMX_VMCS_GUEST_CS_BASE:
4351	case VMX_VMCS_GUEST_DS_BASE:
4352	case VMX_VMCS_GUEST_ES_BASE:
4353	case VMX_VMCS_GUEST_FS_BASE:
4354	case VMX_VMCS_GUEST_GS_BASE:
4355	case VMX_VMCS_GUEST_SS_BASE:
4356	case VMX_VMCS_GUEST_LDTR_BASE:
4357	case VMX_VMCS_GUEST_TR_BASE:
4358	case VMX_VMCS_GUEST_CR3:
4359	return true;
4360	}
4361	return false;
4362	}
4363
4364	static bool hmR0VmxIsValidReadFieldInCache(uint32_t idxField)
4365	{
4366	switch (idxField)
4367	{
4368	/* Read-only fields. */
4369	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4370	return true;
4371	}
4372	/* Remaining readable fields should also be writable. */
4373	return hmR0VmxIsValidWriteFieldInCache(idxField);
4374	}
4375	# endif /* VBOX_STRICT */
4376
4377
4378	/**
4379	* Executes the specified handler in 64-bit mode.
4380	*
4381	* @returns VBox status code (no informational status codes).
4382	* @param pVCpu The cross context virtual CPU structure.
4383	* @param enmOp The operation to perform.
4384	* @param cParams Number of parameters.
4385	* @param paParam Array of 32-bit parameters.
4386	*/
4387	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
4388	{
4389	AssertPtr(pVCpu);
4390	PVM pVM = pVCpu->CTX_SUFF(pVM);
4391	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4392	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4393	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
4394	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
4395
4396	#ifdef VBOX_STRICT
4397	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
4398	Assert(hmR0VmxIsValidWriteFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
4399
4400	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
4401	Assert(hmR0VmxIsValidReadFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
4402	#endif
4403
4404	/* Disable interrupts. */
4405	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
4406
4407	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4408	RTCPUID idHostCpu = RTMpCpuId();
4409	CPUMR0SetLApic(pVCpu, idHostCpu);
4410	#endif
4411
4412	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4413
4414	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4415	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4416
4417	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4418	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4419	hmR0VmxClearVmcs(pVmcsInfo);
4420
4421	/* Leave VMX root mode and disable VMX. */
4422	VMXDisable();
4423	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4424
4425	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4426	CPUMSetHyperEIP(pVCpu, enmOp);
4427	for (int i = (int)cParams - 1; i >= 0; i--)
4428	CPUMPushHyper(pVCpu, paParam[i]);
4429
4430	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4431
4432	/* Call the switcher. */
4433	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
4434	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4435
4436	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
4437	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
4438
4439	/* Re-enter VMX root mode. */
4440	int rc2 = VMXEnable(HCPhysCpuPage);
4441	if (RT_FAILURE(rc2))
4442	{
4443	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4444	ASMSetFlags(fOldEFlags);
4445	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
4446	return rc2;
4447	}
4448
4449	/* Restore the VMCS as the current VMCS. */
4450	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
4451	AssertRC(rc2);
4452	Assert(!(ASMGetFlags() & X86_EFL_IF));
4453	ASMSetFlags(fOldEFlags);
4454	return rc;
4455	}
4456
4457
4458	/**
4459	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4460	* supporting 64-bit guests.
4461	*
4462	* @returns VBox status code.
4463	* @param fResume Whether to VMLAUNCH or VMRESUME.
4464	* @param pCtx Pointer to the guest-CPU context.
4465	* @param pCache Pointer to the VMCS batch cache.
4466	* @param pVM The cross context VM structure.
4467	* @param pVCpu The cross context virtual CPU structure.
4468	*/
4469	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4470	{
4471	NOREF(fResume);
4472
4473	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4474	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4475	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4476
4477	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4478	pCache->uPos = 1;
4479	pCache->interPD = PGMGetInterPaeCR3(pVM);
4480	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4481	#endif
4482
4483	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4484	pCache->TestIn.HCPhysCpuPage = 0;
4485	pCache->TestIn.HCPhysVmcs = 0;
4486	pCache->TestIn.pCache = 0;
4487	pCache->TestOut.HCPhysVmcs = 0;
4488	pCache->TestOut.pCache = 0;
4489	pCache->TestOut.pCtx = 0;
4490	pCache->TestOut.eflags = 0;
4491	#else
4492	NOREF(pCache);
4493	#endif
4494
4495	uint32_t aParam[10];
4496	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4497	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4498	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4499	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4500	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4501	aParam[5] = 0;
4502	aParam[6] = VM_RC_ADDR(pVM, pVM);
4503	aParam[7] = 0;
4504	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4505	aParam[9] = 0;
4506
4507	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4508	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4509	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4510	#endif
4511	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4512
4513	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4514	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4515	Assert(pCtx->dr[4] == 10);
4516	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4517	#endif
4518
4519	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4520	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4521	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4522	pVmcsInfo->HCPhysVmcs));
4523	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4524	pCache->TestOut.HCPhysVmcs));
4525	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4526	pCache->TestOut.pCache));
4527	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4528	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4529	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4530	pCache->TestOut.pCtx));
4531	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4532	#endif
4533	NOREF(pCtx);
4534	return rc;
4535	}
4536	#endif
4537
4538
4539	/**
4540	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4541	* the VMCS.
4542	*
4543	* @returns VBox status code.
4544	*/
4545	static int hmR0VmxExportHostControlRegs(void)
4546	{
4547	RTCCUINTREG uReg = ASMGetCR0();
4548	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4549	AssertRCReturn(rc, rc);
4550
4551	uReg = ASMGetCR3();
4552	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4553	AssertRCReturn(rc, rc);
4554
4555	uReg = ASMGetCR4();
4556	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4557	AssertRCReturn(rc, rc);
4558	return rc;
4559	}
4560
4561
4562	/**
4563	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4564	* the host-state area in the VMCS.
4565	*
4566	* @returns VBox status code.
4567	* @param pVCpu The cross context virtual CPU structure.
4568	*/
4569	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4570	{
4571	#if HC_ARCH_BITS == 64
4572	/**
4573	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4574	* requirements. See hmR0VmxExportHostSegmentRegs().
4575	*/
4576	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4577	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4578	{ \
4579	bool fValidSelector = true; \
4580	if ((selValue) & X86_SEL_LDT) \
4581	{ \
4582	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4583	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4584	} \
4585	if (fValidSelector) \
4586	{ \
4587	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4588	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4589	} \
4590	(selValue) = 0; \
4591	}
4592
4593	/*
4594	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4595	* will be messed up. We should -not- save the messed up state without restoring
4596	* the original host-state, see @bugref{7240}.
4597	*
4598	* This apparently can happen (most likely the FPU changes), deal with it rather than
4599	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4600	*/
4601	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4602	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4603	{
4604	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4605	pVCpu->idCpu));
4606	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4607	}
4608	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4609	#else
4610	RT_NOREF(pVCpu);
4611	#endif
4612
4613	/*
4614	* Host DS, ES, FS and GS segment registers.
4615	*/
4616	#if HC_ARCH_BITS == 64
4617	RTSEL uSelDS = ASMGetDS();
4618	RTSEL uSelES = ASMGetES();
4619	RTSEL uSelFS = ASMGetFS();
4620	RTSEL uSelGS = ASMGetGS();
4621	#else
4622	RTSEL uSelDS = 0;
4623	RTSEL uSelES = 0;
4624	RTSEL uSelFS = 0;
4625	RTSEL uSelGS = 0;
4626	#endif
4627
4628	/*
4629	* Host CS and SS segment registers.
4630	*/
4631	RTSEL uSelCS = ASMGetCS();
4632	RTSEL uSelSS = ASMGetSS();
4633
4634	/*
4635	* Host TR segment register.
4636	*/
4637	RTSEL uSelTR = ASMGetTR();
4638
4639	#if HC_ARCH_BITS == 64
4640	/*
4641	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4642	* gain VM-entry and restore them before we get preempted.
4643	*
4644	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4645	*/
4646	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4647	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4648	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4649	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4650	# undef VMXLOCAL_ADJUST_HOST_SEG
4651	#endif
4652
4653	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4654	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4655	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4656	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4657	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4658	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4659	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4660	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4661	Assert(uSelCS);
4662	Assert(uSelTR);
4663
4664	/* Write these host selector fields into the host-state area in the VMCS. */
4665	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4666	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4667	#if HC_ARCH_BITS == 64
4668	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4669	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4670	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4671	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4672	#else
4673	NOREF(uSelDS);
4674	NOREF(uSelES);
4675	NOREF(uSelFS);
4676	NOREF(uSelGS);
4677	#endif
4678	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4679	AssertRCReturn(rc, rc);
4680
4681	/*
4682	* Host GDTR and IDTR.
4683	*/
4684	RTGDTR Gdtr;
4685	RTIDTR Idtr;
4686	RT_ZERO(Gdtr);
4687	RT_ZERO(Idtr);
4688	ASMGetGDTR(&Gdtr);
4689	ASMGetIDTR(&Idtr);
4690	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4691	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4692	AssertRCReturn(rc, rc);
4693
4694	#if HC_ARCH_BITS == 64
4695	/*
4696	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4697	* them to the maximum limit (0xffff) on every VM-exit.
4698	*/
4699	if (Gdtr.cbGdt != 0xffff)
4700	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4701
4702	/*
4703	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4704	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4705	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4706	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4707	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4708	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4709	* at 0xffff on hosts where we are sure it won't cause trouble.
4710	*/
4711	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4712	if (Idtr.cbIdt < 0x0fff)
4713	# else
4714	if (Idtr.cbIdt != 0xffff)
4715	# endif
4716	{
4717	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4718	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4719	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4720	}
4721	#endif
4722
4723	/*
4724	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4725	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4726	* RPL should be too in most cases.
4727	*/
4728	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4729	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4730
4731	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4732	#if HC_ARCH_BITS == 64
4733	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4734
4735	/*
4736	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4737	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4738	* restoration if the host has something else. Task switching is not supported in 64-bit
4739	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4740	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4741	*
4742	* [1] See Intel spec. 3.5 "System Descriptor Types".
4743	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4744	*/
4745	PVM pVM = pVCpu->CTX_SUFF(pVM);
4746	Assert(pDesc->System.u4Type == 11);
4747	if ( pDesc->System.u16LimitLow != 0x67
4748	\|\| pDesc->System.u4LimitHigh)
4749	{
4750	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4751	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4752	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4753	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4754	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4755	}
4756
4757	/*
4758	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4759	*/
4760	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4761	{
4762	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4763	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4764	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4765	{
4766	/* The GDT is read-only but the writable GDT is available. */
4767	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4768	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4769	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4770	AssertRCReturn(rc, rc);
4771	}
4772	}
4773	#else
4774	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4775	#endif
4776	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4777	AssertRCReturn(rc, rc);
4778
4779	/*
4780	* Host FS base and GS base.
4781	*/
4782	#if HC_ARCH_BITS == 64
4783	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4784	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4785	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4786	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4787	AssertRCReturn(rc, rc);
4788
4789	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4790	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4791	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4792	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4793	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4794	#endif
4795	return VINF_SUCCESS;
4796	}
4797
4798
4799	/**
4800	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4801	* host-state area of the VMCS.
4802	*
4803	* These MSRs will be automatically restored on the host after every successful
4804	* VM-exit.
4805	*
4806	* @returns VBox status code.
4807	* @param pVCpu The cross context virtual CPU structure.
4808	*
4809	* @remarks No-long-jump zone!!!
4810	*/
4811	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4812	{
4813	AssertPtr(pVCpu);
4814
4815	/*
4816	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4817	* rather than swapping them on every VM-entry.
4818	*/
4819	hmR0VmxLazySaveHostMsrs(pVCpu);
4820
4821	/*
4822	* Host Sysenter MSRs.
4823	*/
4824	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4825	#if HC_ARCH_BITS == 32
4826	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4827	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4828	#else
4829	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4830	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4831	#endif
4832	AssertRCReturn(rc, rc);
4833
4834	/*
4835	* Host EFER MSR.
4836	*
4837	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4838	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4839	*/
4840	PVM pVM = pVCpu->CTX_SUFF(pVM);
4841	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4842	{
4843	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4844	AssertRCReturn(rc, rc);
4845	}
4846
4847	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4848	* hmR0VmxExportGuestEntryExitCtls(). */
4849
4850	return VINF_SUCCESS;
4851	}
4852
4853
4854	/**
4855	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4856	*
4857	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4858	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4859	*
4860	* @returns true if we need to load guest EFER, false otherwise.
4861	* @param pVCpu The cross context virtual CPU structure.
4862	*
4863	* @remarks Requires EFER, CR4.
4864	* @remarks No-long-jump zone!!!
4865	*/
4866	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4867	{
4868	#ifdef HMVMX_ALWAYS_SWAP_EFER
4869	RT_NOREF(pVCpu);
4870	return true;
4871	#else
4872	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4873	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4874	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4875	if (CPUMIsGuestInLongModeEx(pCtx))
4876	return false;
4877	#endif
4878
4879	PVM pVM = pVCpu->CTX_SUFF(pVM);
4880	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4881	uint64_t const u64GuestEfer = pCtx->msrEFER;
4882
4883	/*
4884	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4885	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4886	*/
4887	if ( CPUMIsGuestInLongModeEx(pCtx)
4888	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4889	return true;
4890
4891	/*
4892	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4893	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4894	*
4895	* See Intel spec. 4.5 "IA-32e Paging".
4896	* See Intel spec. 4.1.1 "Three Paging Modes".
4897	*
4898	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4899	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4900	*/
4901	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4902	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4903	if ( (pCtx->cr4 & X86_CR4_PAE)
4904	&& (pCtx->cr0 & X86_CR0_PG)
4905	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4906	{
4907	/* Assert that host is NX capable. */
4908	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4909	return true;
4910	}
4911
4912	return false;
4913	#endif
4914	}
4915
4916	/**
4917	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4918	* VMCS.
4919	*
4920	* This is typically required when the guest changes paging mode.
4921	*
4922	* @returns VBox status code.
4923	* @param pVCpu The cross context virtual CPU structure.
4924	* @param pVmxTransient The VMX-transient structure.
4925	*
4926	* @remarks Requires EFER.
4927	* @remarks No-long-jump zone!!!
4928	*/
4929	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4930	{
4931	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4932	{
4933	PVM pVM = pVCpu->CTX_SUFF(pVM);
4934	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4935
4936	/*
4937	* VM-entry controls.
4938	*/
4939	{
4940	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4941	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4942
4943	/*
4944	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4945	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4946	*
4947	* For nested-guests, this is a mandatory VM-entry control. It's also
4948	* required because we do not want to leak host bits to the nested-guest.
4949	*/
4950	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4951
4952	/*
4953	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4954	*
4955	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4956	* required to get the nested-guest working with hardware-assisted VMX execution.
4957	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4958	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4959	* here rather than while merging the guest VMCS controls.
4960	*/
4961	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4962	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4963	else
4964	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4965
4966	/*
4967	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4968	*
4969	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4970	* regardless of whether the nested-guest VMCS specifies it because we are free to
4971	* load whatever MSRs we require and we do not need to modify the guest visible copy
4972	* of the VM-entry MSR load area.
4973	*/
4974	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4975	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4976	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4977	else
4978	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4979
4980	/*
4981	* The following should -not- be set (since we're not in SMM mode):
4982	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4983	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4984	*/
4985
4986	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4987	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4988
4989	if ((fVal & fZap) == fVal)
4990	{ /* likely */ }
4991	else
4992	{
4993	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4994	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4995	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4996	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4997	}
4998
4999	/* Commit it to the VMCS. */
5000	if (pVmcsInfo->u32EntryCtls != fVal)
5001	{
5002	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
5003	AssertRCReturn(rc, rc);
5004	pVmcsInfo->u32EntryCtls = fVal;
5005	}
5006	}
5007
5008	/*
5009	* VM-exit controls.
5010	*/
5011	{
5012	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
5013	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
5014
5015	/*
5016	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
5017	* supported the 1-setting of this bit.
5018	*
5019	* For nested-guests, we set the "save debug controls" as the converse
5020	* "load debug controls" is mandatory for nested-guests anyway.
5021	*/
5022	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
5023
5024	/*
5025	* Set the host long mode active (EFER.LMA) bit (which Intel calls
5026	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
5027	* host EFER.LMA and EFER.LME bit to this value. See assertion in
5028	* hmR0VmxExportHostMsrs().
5029	*
5030	* For nested-guests, we always set this bit as we do not support 32-bit
5031	* hosts.
5032	*/
5033	#if HC_ARCH_BITS == 64
5034	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5035	#else
5036	Assert(!pVmxTransient->fIsNestedGuest);
5037	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
5038	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
5039	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
5040	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
5041	{
5042	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
5043	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5044	}
5045	else
5046	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
5047	#endif
5048
5049	/*
5050	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
5051	*
5052	* For nested-guests, we should use the "save IA32_EFER" control if we also
5053	* used the "load IA32_EFER" control while exporting VM-entry controls.
5054	*/
5055	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
5056	&& hmR0VmxShouldSwapEferMsr(pVCpu))
5057	{
5058	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
5059	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
5060	}
5061
5062	/*
5063	* Enable saving of the VMX-preemption timer value on VM-exit.
5064	* For nested-guests, currently not exposed/used.
5065	*/
5066	if ( pVM->hm.s.vmx.fUsePreemptTimer
5067	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
5068	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
5069
5070	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
5071	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
5072
5073	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
5074	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
5075	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
5076
5077	if ((fVal & fZap) == fVal)
5078	{ /* likely */ }
5079	else
5080	{
5081	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
5082	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
5083	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
5084	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5085	}
5086
5087	/* Commit it to the VMCS. */
5088	if (pVmcsInfo->u32ExitCtls != fVal)
5089	{
5090	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5091	AssertRCReturn(rc, rc);
5092	pVmcsInfo->u32ExitCtls = fVal;
5093	}
5094	}
5095
5096	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5097	}
5098	return VINF_SUCCESS;
5099	}
5100
5101
5102	/**
5103	* Sets the TPR threshold in the VMCS.
5104	*
5105	* @returns VBox status code.
5106	* @param pVCpu The cross context virtual CPU structure.
5107	* @param pVmcsInfo The VMCS info. object.
5108	* @param u32TprThreshold The TPR threshold (task-priority class only).
5109	*/
5110	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5111	{
5112	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5113	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5114	RT_NOREF2(pVCpu, pVmcsInfo);
5115	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5116	}
5117
5118
5119	/**
5120	* Exports the guest APIC TPR state into the VMCS.
5121	*
5122	* @returns VBox status code.
5123	* @param pVCpu The cross context virtual CPU structure.
5124	* @param pVmxTransient The VMX-transient structure.
5125	*
5126	* @remarks No-long-jump zone!!!
5127	*/
5128	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5129	{
5130	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5131	{
5132	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5133
5134	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5135	if (!pVmxTransient->fIsNestedGuest)
5136	{
5137	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5138	&& APICIsEnabled(pVCpu))
5139	{
5140	/*
5141	* Setup TPR shadowing.
5142	*/
5143	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5144	{
5145	bool fPendingIntr = false;
5146	uint8_t u8Tpr = 0;
5147	uint8_t u8PendingIntr = 0;
5148	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5149	AssertRCReturn(rc, rc);
5150
5151	/*
5152	* If there are interrupts pending but masked by the TPR, instruct VT-x to
5153	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5154	* priority of the pending interrupt so we can deliver the interrupt. If there
5155	* are no interrupts pending, set threshold to 0 to not cause any
5156	* TPR-below-threshold VM-exits.
5157	*/
5158	Assert(pVmcsInfo->pbVirtApic);
5159	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
5160	uint32_t u32TprThreshold = 0;
5161	if (fPendingIntr)
5162	{
5163	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5164	(which is the Task-Priority Class). */
5165	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5166	const uint8_t u8TprPriority = u8Tpr >> 4;
5167	if (u8PendingPriority <= u8TprPriority)
5168	u32TprThreshold = u8PendingPriority;
5169	}
5170
5171	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
5172	AssertRCReturn(rc, rc);
5173	}
5174	}
5175	}
5176	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5177	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5178	}
5179	return VINF_SUCCESS;
5180	}
5181
5182
5183	/**
5184	* Gets the guest interruptibility-state.
5185	*
5186	* @returns Guest's interruptibility-state.
5187	* @param pVCpu The cross context virtual CPU structure.
5188	* @param pVmxTransient The VMX-transient structure.
5189	*
5190	* @remarks No-long-jump zone!!!
5191	*/
5192	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5193	{
5194	/*
5195	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5196	*/
5197	uint32_t fIntrState = 0;
5198	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5199	{
5200	/* If inhibition is active, RIP and RFLAGS should've been updated
5201	(i.e. read previously from the VMCS or from ring-3). */
5202	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5203	#ifdef VBOX_STRICT
5204	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
5205	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5206	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
5207	#endif
5208	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5209	{
5210	if (pCtx->eflags.Bits.u1IF)
5211	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5212	else
5213	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5214	}
5215	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5216	{
5217	/*
5218	* We can clear the inhibit force flag as even if we go back to the recompiler
5219	* without executing guest code in VT-x, the flag's condition to be cleared is
5220	* met and thus the cleared state is correct.
5221	*/
5222	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5223	}
5224	}
5225
5226	/*
5227	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5228	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5229	* setting this would block host-NMIs and IRET will not clear the blocking.
5230	*
5231	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5232	*
5233	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5234	*/
5235	if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
5236	&& CPUMIsGuestNmiBlocking(pVCpu))
5237	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5238
5239	return fIntrState;
5240	}
5241
5242
5243	/**
5244	* Exports the exception intercepts required for guest execution in the VMCS.
5245	*
5246	* @returns VBox status code.
5247	* @param pVCpu The cross context virtual CPU structure.
5248	* @param pVmxTransient The VMX-transient structure.
5249	*
5250	* @remarks No-long-jump zone!!!
5251	*/
5252	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5253	{
5254	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5255	{
5256	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5257	if ( !pVmxTransient->fIsNestedGuest
5258	&& pVCpu->hm.s.fGIMTrapXcptUD)
5259	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5260	else
5261	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5262
5263	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5264	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5265	}
5266	return VINF_SUCCESS;
5267	}
5268
5269
5270	/**
5271	* Exports the guest's RIP into the guest-state area in the VMCS.
5272	*
5273	* @returns VBox status code.
5274	* @param pVCpu The cross context virtual CPU structure.
5275	*
5276	* @remarks No-long-jump zone!!!
5277	*/
5278	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5279	{
5280	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5281	{
5282	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5283
5284	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5285	AssertRCReturn(rc, rc);
5286
5287	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5288	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5289	}
5290	return VINF_SUCCESS;
5291	}
5292
5293
5294	/**
5295	* Exports the guest's RSP into the guest-state area in the VMCS.
5296	*
5297	* @returns VBox status code.
5298	* @param pVCpu The cross context virtual CPU structure.
5299	*
5300	* @remarks No-long-jump zone!!!
5301	*/
5302	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5303	{
5304	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5305	{
5306	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5307
5308	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5309	AssertRCReturn(rc, rc);
5310
5311	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5312	}
5313	return VINF_SUCCESS;
5314	}
5315
5316
5317	/**
5318	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5319	*
5320	* @returns VBox status code.
5321	* @param pVCpu The cross context virtual CPU structure.
5322	* @param pVmxTransient The VMX-transient structure.
5323	*
5324	* @remarks No-long-jump zone!!!
5325	*/
5326	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5327	{
5328	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5329	{
5330	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5331
5332	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5333	Let us assert it as such and use 32-bit VMWRITE. */
5334	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5335	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5336	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5337	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5338
5339	/*
5340	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5341	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5342	* can run the real-mode guest code under Virtual 8086 mode.
5343	*/
5344	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5345	if (pVmcsInfo->RealMode.fRealOnV86Active)
5346	{
5347	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5348	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5349	Assert(!pVmxTransient->fIsNestedGuest);
5350	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5351	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5352	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5353	}
5354
5355	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5356	AssertRCReturn(rc, rc);
5357
5358	/*
5359	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5360	*
5361	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5362	* through the hypervisor debugger using EFLAGS.TF.
5363	*/
5364	if ( !pVmxTransient->fIsNestedGuest
5365	&& !pVCpu->hm.s.fSingleInstruction
5366	&& fEFlags.Bits.u1TF)
5367	{
5368	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5369	* premature trips to ring-3 esp since IEM does not yet handle it. */
5370	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5371	AssertRCReturn(rc, rc);
5372	}
5373	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5374	* nested-guest VMCS. */
5375
5376	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5377	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5378	}
5379	return VINF_SUCCESS;
5380	}
5381
5382
5383	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5384	/**
5385	* Copies the nested-guest VMCS to the shadow VMCS.
5386	*
5387	* @returns VBox status code.
5388	* @param pVCpu The cross context virtual CPU structure.
5389	* @param pVmcsInfo The VMCS info. object.
5390	*
5391	* @remarks No-long-jump zone!!!
5392	*/
5393	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5394	{
5395	PVM pVM = pVCpu->CTX_SUFF(pVM);
5396	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5397
5398	/*
5399	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5400	* current VMCS, as we may try saving guest lazy MSRs.
5401	*
5402	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5403	* calling the import VMCS code which is currently performing the guest MSR reads
5404	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5405	* and the rest of the VMX leave session machinery.
5406	*/
5407	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5408
5409	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5410	if (RT_SUCCESS(rc))
5411	{
5412	/*
5413	* Copy all guest read/write VMCS fields.
5414	*
5415	* We don't check for VMWRITE failures here for performance reasons and
5416	* because they are not expected to fail, barring irrecoverable conditions
5417	* like hardware errors.
5418	*/
5419	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5420	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5421	{
5422	uint64_t u64Val;
5423	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5424	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5425	VMXWriteVmcs64(uVmcsField, u64Val);
5426	}
5427
5428	/*
5429	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5430	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5431	*/
5432	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5433	{
5434	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5435	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5436	{
5437	uint64_t u64Val;
5438	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5439	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5440	VMXWriteVmcs64(uVmcsField, u64Val);
5441	}
5442	}
5443
5444	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5445	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5446	}
5447
5448	ASMSetFlags(fEFlags);
5449	return rc;
5450	}
5451
5452
5453	/**
5454	* Copies the shadow VMCS to the nested-guest VMCS.
5455	*
5456	* @returns VBox status code.
5457	* @param pVCpu The cross context virtual CPU structure.
5458	* @param pVmcsInfo The VMCS info. object.
5459	*
5460	* @remarks Called with interrupts disabled.
5461	*/
5462	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5463	{
5464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5465	PVM pVM = pVCpu->CTX_SUFF(pVM);
5466	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5467
5468	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5469	if (RT_SUCCESS(rc))
5470	{
5471	/*
5472	* Copy guest read/write fields from the shadow VMCS.
5473	* Guest read-only fields cannot be modified, so no need to copy them.
5474	*
5475	* We don't check for VMREAD failures here for performance reasons and
5476	* because they are not expected to fail, barring irrecoverable conditions
5477	* like hardware errors.
5478	*/
5479	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5480	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5481	{
5482	uint64_t u64Val;
5483	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5484	VMXReadVmcs64(uVmcsField, &u64Val);
5485	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5486	}
5487
5488	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5489	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5490	}
5491	return rc;
5492	}
5493
5494
5495	/**
5496	* Enables VMCS shadowing for the given VMCS info. object.
5497	*
5498	* @param pVCpu The cross context virtual CPU structure.
5499	* @param pVmcsInfo The VMCS info. object.
5500	*
5501	* @remarks No-long-jump zone!!!
5502	*/
5503	static void hmR0VmxEnableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5504	{
5505	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5506
5507	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5508	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5509	{
5510	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5511	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5512	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5513	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5514	AssertRC(rc);
5515
5516	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5517	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5518	Log4Func(("Enabled\n"));
5519	}
5520	}
5521
5522
5523	/**
5524	* Disables VMCS shadowing for the given VMCS info. object.
5525	*
5526	* @param pVCpu The cross context virtual CPU structure.
5527	* @param pVmcsInfo The VMCS info. object.
5528	*
5529	* @remarks No-long-jump zone!!!
5530	*/
5531	static void hmR0VmxDisableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5532	{
5533	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5534
5535	/*
5536	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5537	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5538	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5539	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5540	*
5541	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5542	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5543	*/
5544	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5545	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5546	{
5547	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5548	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5549	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5550	AssertRC(rc);
5551
5552	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5553	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5554	Log4Func(("Disabled\n"));
5555	}
5556	}
5557	#endif
5558
5559
5560	/**
5561	* Exports the guest hardware-virtualization state.
5562	*
5563	* @returns VBox status code.
5564	* @param pVCpu The cross context virtual CPU structure.
5565	* @param pVmxTransient The VMX-transient structure.
5566	*
5567	* @remarks No-long-jump zone!!!
5568	*/
5569	static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5570	{
5571	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5572	{
5573	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5574	/*
5575	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5576	* VMCS shadowing.
5577	*/
5578	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5579	{
5580	/*
5581	* If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5582	* copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5583	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5584	*
5585	* We check for VMX root mode here in case the guest executes VMXOFF without
5586	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5587	* not clear the current VMCS pointer.
5588	*/
5589	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5590	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5591	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5592	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5593	{
5594	/* Paranoia. */
5595	Assert(!pVmxTransient->fIsNestedGuest);
5596
5597	/*
5598	* For performance reasons, also check if the guest hypervisor's current VMCS
5599	* was newly loaded or modified before copying it to the shadow VMCS.
5600	*/
5601	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5602	{
5603	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5604	AssertRCReturn(rc, rc);
5605	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5606	}
5607	hmR0VmxEnableVmcsShadowing(pVCpu, pVmcsInfo);
5608	}
5609	else
5610	hmR0VmxDisableVmcsShadowing(pVCpu, pVmcsInfo);
5611	}
5612	#else
5613	NOREF(pVmxTransient);
5614	#endif
5615	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5616	}
5617	return VINF_SUCCESS;
5618	}
5619
5620
5621	/**
5622	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5623	*
5624	* The guest FPU state is always pre-loaded hence we don't need to bother about
5625	* sharing FPU related CR0 bits between the guest and host.
5626	*
5627	* @returns VBox status code.
5628	* @param pVCpu The cross context virtual CPU structure.
5629	* @param pVmxTransient The VMX-transient structure.
5630	*
5631	* @remarks No-long-jump zone!!!
5632	*/
5633	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5634	{
5635	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5636	{
5637	PVM pVM = pVCpu->CTX_SUFF(pVM);
5638	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5639
5640	/*
5641	* Figure out fixed CR0 bits in VMX operation.
5642	*/
5643	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5644	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5645	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5646	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5647	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5648	else
5649	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5650
5651	if (!pVmxTransient->fIsNestedGuest)
5652	{
5653	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5654	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5655	uint64_t const u64ShadowCr0 = u64GuestCr0;
5656	Assert(!RT_HI_U32(u64GuestCr0));
5657
5658	/*
5659	* Setup VT-x's view of the guest CR0.
5660	*/
5661	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5662	if (pVM->hm.s.fNestedPaging)
5663	{
5664	if (CPUMIsGuestPagingEnabled(pVCpu))
5665	{
5666	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5667	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5668	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5669	}
5670	else
5671	{
5672	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5673	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5674	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5675	}
5676
5677	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5678	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5679	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5680	}
5681	else
5682	{
5683	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5684	u64GuestCr0 \|= X86_CR0_WP;
5685	}
5686
5687	/*
5688	* Guest FPU bits.
5689	*
5690	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5691	* using CR0.TS.
5692	*
5693	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5694	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5695	*/
5696	u64GuestCr0 \|= X86_CR0_NE;
5697
5698	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5699	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5700
5701	/*
5702	* Update exception intercepts.
5703	*/
5704	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5705	if (pVmcsInfo->RealMode.fRealOnV86Active)
5706	{
5707	Assert(PDMVmmDevHeapIsEnabled(pVM));
5708	Assert(pVM->hm.s.vmx.pRealModeTSS);
5709	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5710	}
5711	else
5712	{
5713	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5714	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5715	if (fInterceptMF)
5716	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5717	}
5718
5719	/* Additional intercepts for debugging, define these yourself explicitly. */
5720	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5721	uXcptBitmap \|= 0
5722	\| RT_BIT(X86_XCPT_BP)
5723	\| RT_BIT(X86_XCPT_DE)
5724	\| RT_BIT(X86_XCPT_NM)
5725	\| RT_BIT(X86_XCPT_TS)
5726	\| RT_BIT(X86_XCPT_UD)
5727	\| RT_BIT(X86_XCPT_NP)
5728	\| RT_BIT(X86_XCPT_SS)
5729	\| RT_BIT(X86_XCPT_GP)
5730	\| RT_BIT(X86_XCPT_PF)
5731	\| RT_BIT(X86_XCPT_MF)
5732	;
5733	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5734	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5735	#endif
5736	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5737	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5738	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5739
5740	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5741	u64GuestCr0 \|= fSetCr0;
5742	u64GuestCr0 &= fZapCr0;
5743	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5744
5745	/* Commit the CR0 and related fields to the guest VMCS. */
5746	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5747	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5748	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5749	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5750	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5751	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5752	AssertRCReturn(rc, rc);
5753
5754	/* Update our caches. */
5755	pVmcsInfo->u32ProcCtls = uProcCtls;
5756	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5757
5758	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5759	}
5760	else
5761	{
5762	/*
5763	* With nested-guests, we may have extended the guest/host mask here since we
5764	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5765	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5766	* originally supplied. We must copy those bits from the nested-guest CR0 into
5767	* the nested-guest CR0 read-shadow.
5768	*/
5769	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5770	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5771	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5772	Assert(!RT_HI_U32(u64GuestCr0));
5773	Assert(u64GuestCr0 & X86_CR0_NE);
5774
5775	/*
5776	* Apply the hardware specified fixed CR0 bits and enable caching.
5777	* Note! We could be altering our VMX emulation's fixed bits. We thus
5778	* need to re-apply them while importing CR0.
5779	*/
5780	u64GuestCr0 \|= fSetCr0;
5781	u64GuestCr0 &= fZapCr0;
5782	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5783
5784	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5785	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5786	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5787	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5788	AssertRCReturn(rc, rc);
5789
5790	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5791	}
5792
5793	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5794	}
5795
5796	return VINF_SUCCESS;
5797	}
5798
5799
5800	/**
5801	* Exports the guest control registers (CR3, CR4) into the guest-state area
5802	* in the VMCS.
5803	*
5804	* @returns VBox strict status code.
5805	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5806	* without unrestricted guest access and the VMMDev is not presently
5807	* mapped (e.g. EFI32).
5808	*
5809	* @param pVCpu The cross context virtual CPU structure.
5810	* @param pVmxTransient The VMX-transient structure.
5811	*
5812	* @remarks No-long-jump zone!!!
5813	*/
5814	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5815	{
5816	int rc = VINF_SUCCESS;
5817	PVM pVM = pVCpu->CTX_SUFF(pVM);
5818
5819	/*
5820	* Guest CR2.
5821	* It's always loaded in the assembler code. Nothing to do here.
5822	*/
5823
5824	/*
5825	* Guest CR3.
5826	*/
5827	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5828	{
5829	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5830
5831	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5832	if (pVM->hm.s.fNestedPaging)
5833	{
5834	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5835	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5836
5837	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5838	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5839	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5840	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5841
5842	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5843	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5844	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5845
5846	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5847	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5848	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5849	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5850	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5851	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5852	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5853
5854	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5855	AssertRCReturn(rc, rc);
5856
5857	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5858	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5859	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5860	{
5861	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5862	if (CPUMIsGuestInPAEModeEx(pCtx))
5863	{
5864	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5865	AssertRCReturn(rc, rc);
5866	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5867	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5868	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5869	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5870	AssertRCReturn(rc, rc);
5871	}
5872
5873	/*
5874	* The guest's view of its CR3 is unblemished with nested paging when the
5875	* guest is using paging or we have unrestricted guest execution to handle
5876	* the guest when it's not using paging.
5877	*/
5878	GCPhysGuestCr3 = pCtx->cr3;
5879	}
5880	else
5881	{
5882	/*
5883	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5884	* thinks it accesses physical memory directly, we use our identity-mapped
5885	* page table to map guest-linear to guest-physical addresses. EPT takes care
5886	* of translating it to host-physical addresses.
5887	*/
5888	RTGCPHYS GCPhys;
5889	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5890
5891	/* We obtain it here every time as the guest could have relocated this PCI region. */
5892	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5893	if (RT_SUCCESS(rc))
5894	{ /* likely */ }
5895	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5896	{
5897	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5898	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5899	}
5900	else
5901	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5902
5903	GCPhysGuestCr3 = GCPhys;
5904	}
5905
5906	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5907	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5908	AssertRCReturn(rc, rc);
5909	}
5910	else
5911	{
5912	/* Non-nested paging case, just use the hypervisor's CR3. */
5913	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5914
5915	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5916	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5917	AssertRCReturn(rc, rc);
5918	}
5919
5920	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5921	}
5922
5923	/*
5924	* Guest CR4.
5925	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5926	*/
5927	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5928	{
5929	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5930	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5931
5932	/*
5933	* Figure out fixed CR4 bits in VMX operation.
5934	*/
5935	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5936	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5937	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5938
5939	/*
5940	* With nested-guests, we may have extended the guest/host mask here (since we
5941	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5942	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5943	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5944	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5945	*/
5946	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5947	uint64_t u64GuestCr4 = pCtx->cr4;
5948	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5949	? pCtx->cr4
5950	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5951	Assert(!RT_HI_U32(u64GuestCr4));
5952
5953	/*
5954	* Setup VT-x's view of the guest CR4.
5955	*
5956	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5957	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5958	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5959	*
5960	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5961	*/
5962	if (pVmcsInfo->RealMode.fRealOnV86Active)
5963	{
5964	Assert(pVM->hm.s.vmx.pRealModeTSS);
5965	Assert(PDMVmmDevHeapIsEnabled(pVM));
5966	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5967	}
5968
5969	if (pVM->hm.s.fNestedPaging)
5970	{
5971	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5972	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5973	{
5974	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5975	u64GuestCr4 \|= X86_CR4_PSE;
5976	/* Our identity mapping is a 32-bit page directory. */
5977	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5978	}
5979	/* else use guest CR4.*/
5980	}
5981	else
5982	{
5983	Assert(!pVmxTransient->fIsNestedGuest);
5984
5985	/*
5986	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5987	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5988	*/
5989	switch (pVCpu->hm.s.enmShadowMode)
5990	{
5991	case PGMMODE_REAL: /* Real-mode. */
5992	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5993	case PGMMODE_32_BIT: /* 32-bit paging. */
5994	{
5995	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5996	break;
5997	}
5998
5999	case PGMMODE_PAE: /* PAE paging. */
6000	case PGMMODE_PAE_NX: /* PAE paging with NX. */
6001	{
6002	u64GuestCr4 \|= X86_CR4_PAE;
6003	break;
6004	}
6005
6006	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
6007	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
6008	#ifdef VBOX_ENABLE_64_BITS_GUESTS
6009	break;
6010	#endif
6011	default:
6012	AssertFailed();
6013	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6014	}
6015	}
6016
6017	/*
6018	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
6019	* Note! For nested-guests, we could be altering our VMX emulation's
6020	* fixed bits. We thus need to re-apply them while importing CR4.
6021	*/
6022	u64GuestCr4 \|= fSetCr4;
6023	u64GuestCr4 &= fZapCr4;
6024
6025	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
6026	/** @todo Fix to 64-bit when we drop 32-bit. */
6027	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
6028	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
6029	AssertRCReturn(rc, rc);
6030
6031	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
6032	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6033
6034	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
6035
6036	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
6037	}
6038	return rc;
6039	}
6040
6041
6042	/**
6043	* Exports the guest debug registers into the guest-state area in the VMCS.
6044	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
6045	*
6046	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
6047	*
6048	* @returns VBox status code.
6049	* @param pVCpu The cross context virtual CPU structure.
6050	* @param pVmxTransient The VMX-transient structure.
6051	*
6052	* @remarks No-long-jump zone!!!
6053	*/
6054	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6055	{
6056	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6057
6058	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
6059	* stepping. */
6060	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6061	if (pVmxTransient->fIsNestedGuest)
6062	{
6063	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
6064	AssertRCReturn(rc, rc);
6065	return VINF_SUCCESS;
6066	}
6067
6068	#ifdef VBOX_STRICT
6069	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
6070	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
6071	{
6072	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
6073	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
6074	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
6075	}
6076	#endif
6077
6078	bool fSteppingDB = false;
6079	bool fInterceptMovDRx = false;
6080	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6081	if (pVCpu->hm.s.fSingleInstruction)
6082	{
6083	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
6084	PVM pVM = pVCpu->CTX_SUFF(pVM);
6085	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
6086	{
6087	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
6088	Assert(fSteppingDB == false);
6089	}
6090	else
6091	{
6092	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
6093	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
6094	pVCpu->hm.s.fClearTrapFlag = true;
6095	fSteppingDB = true;
6096	}
6097	}
6098
6099	uint32_t u32GuestDr7;
6100	if ( fSteppingDB
6101	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
6102	{
6103	/*
6104	* Use the combined guest and host DRx values found in the hypervisor register set
6105	* because the hypervisor debugger has breakpoints active or someone is single stepping
6106	* on the host side without a monitor trap flag.
6107	*
6108	* Note! DBGF expects a clean DR6 state before executing guest code.
6109	*/
6110	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6111	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6112	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
6113	{
6114	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6115	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
6116	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
6117	}
6118	else
6119	#endif
6120	if (!CPUMIsHyperDebugStateActive(pVCpu))
6121	{
6122	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6123	Assert(CPUMIsHyperDebugStateActive(pVCpu));
6124	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6125	}
6126
6127	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
6128	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
6129	pVCpu->hm.s.fUsingHyperDR7 = true;
6130	fInterceptMovDRx = true;
6131	}
6132	else
6133	{
6134	/*
6135	* If the guest has enabled debug registers, we need to load them prior to
6136	* executing guest code so they'll trigger at the right time.
6137	*/
6138	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
6139	{
6140	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6141	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6142	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
6143	{
6144	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6145	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
6146	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
6147	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6148	}
6149	else
6150	#endif
6151	if (!CPUMIsGuestDebugStateActive(pVCpu))
6152	{
6153	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6154	Assert(CPUMIsGuestDebugStateActive(pVCpu));
6155	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6156	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6157	}
6158	Assert(!fInterceptMovDRx);
6159	}
6160	/*
6161	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
6162	* must intercept #DB in order to maintain a correct DR6 guest value, and
6163	* because we need to intercept it to prevent nested #DBs from hanging the
6164	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
6165	*/
6166	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6167	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
6168	&& !CPUMIsGuestDebugStateActive(pVCpu))
6169	#else
6170	else if (!CPUMIsGuestDebugStateActive(pVCpu))
6171	#endif
6172	{
6173	fInterceptMovDRx = true;
6174	}
6175
6176	/* Update DR7 with the actual guest value. */
6177	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
6178	pVCpu->hm.s.fUsingHyperDR7 = false;
6179	}
6180
6181	if (fInterceptMovDRx)
6182	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
6183	else
6184	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
6185
6186	/*
6187	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
6188	* monitor-trap flag and update our cache.
6189	*/
6190	if (uProcCtls != pVmcsInfo->u32ProcCtls)
6191	{
6192	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6193	AssertRCReturn(rc2, rc2);
6194	pVmcsInfo->u32ProcCtls = uProcCtls;
6195	}
6196
6197	/*
6198	* Update guest DR7.
6199	*/
6200	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
6201	AssertRCReturn(rc, rc);
6202
6203	/*
6204	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6205	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6206	*
6207	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6208	*/
6209	if (fSteppingDB)
6210	{
6211	Assert(pVCpu->hm.s.fSingleInstruction);
6212	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6213
6214	uint32_t fIntrState = 0;
6215	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6216	AssertRCReturn(rc, rc);
6217
6218	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6219	{
6220	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6221	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6222	AssertRCReturn(rc, rc);
6223	}
6224	}
6225
6226	return VINF_SUCCESS;
6227	}
6228
6229
6230	#ifdef VBOX_STRICT
6231	/**
6232	* Strict function to validate segment registers.
6233	*
6234	* @param pVCpu The cross context virtual CPU structure.
6235	* @param pVmcsInfo The VMCS info. object.
6236	*
6237	* @remarks Will import guest CR0 on strict builds during validation of
6238	* segments.
6239	*/
6240	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
6241	{
6242	/*
6243	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6244	*
6245	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6246	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6247	* unusable bit and doesn't change the guest-context value.
6248	*/
6249	PVM pVM = pVCpu->CTX_SUFF(pVM);
6250	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6251	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6252	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6253	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6254	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6255	{
6256	/* Protected mode checks */
6257	/* CS */
6258	Assert(pCtx->cs.Attr.n.u1Present);
6259	Assert(!(pCtx->cs.Attr.u & 0xf00));
6260	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6261	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6262	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6263	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6264	\|\| (pCtx->cs.Attr.n.u1Granularity));
6265	/* CS cannot be loaded with NULL in protected mode. */
6266	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6267	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6268	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6269	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6270	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6271	else
6272	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6273	/* SS */
6274	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6275	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6276	if ( !(pCtx->cr0 & X86_CR0_PE)
6277	\|\| pCtx->cs.Attr.n.u4Type == 3)
6278	{
6279	Assert(!pCtx->ss.Attr.n.u2Dpl);
6280	}
6281	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6282	{
6283	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6284	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6285	Assert(pCtx->ss.Attr.n.u1Present);
6286	Assert(!(pCtx->ss.Attr.u & 0xf00));
6287	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6288	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6289	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6290	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6291	\|\| (pCtx->ss.Attr.n.u1Granularity));
6292	}
6293	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6294	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6295	{
6296	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6297	Assert(pCtx->ds.Attr.n.u1Present);
6298	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6299	Assert(!(pCtx->ds.Attr.u & 0xf00));
6300	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6301	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6302	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6303	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6304	\|\| (pCtx->ds.Attr.n.u1Granularity));
6305	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6306	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6307	}
6308	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6309	{
6310	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6311	Assert(pCtx->es.Attr.n.u1Present);
6312	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6313	Assert(!(pCtx->es.Attr.u & 0xf00));
6314	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6315	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6316	\|\| !(pCtx->es.Attr.n.u1Granularity));
6317	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6318	\|\| (pCtx->es.Attr.n.u1Granularity));
6319	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6320	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6321	}
6322	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6323	{
6324	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6325	Assert(pCtx->fs.Attr.n.u1Present);
6326	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6327	Assert(!(pCtx->fs.Attr.u & 0xf00));
6328	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6329	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6330	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6331	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6332	\|\| (pCtx->fs.Attr.n.u1Granularity));
6333	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6334	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6335	}
6336	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6337	{
6338	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6339	Assert(pCtx->gs.Attr.n.u1Present);
6340	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6341	Assert(!(pCtx->gs.Attr.u & 0xf00));
6342	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6343	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6344	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6345	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6346	\|\| (pCtx->gs.Attr.n.u1Granularity));
6347	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6348	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6349	}
6350	/* 64-bit capable CPUs. */
6351	# if HC_ARCH_BITS == 64
6352	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6353	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6354	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6355	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6356	# endif
6357	}
6358	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6359	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6360	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6361	{
6362	/* Real and v86 mode checks. */
6363	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6364	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6365	if (pVmcsInfo->RealMode.fRealOnV86Active)
6366	{
6367	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6368	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6369	}
6370	else
6371	{
6372	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6373	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6374	}
6375
6376	/* CS */
6377	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6378	Assert(pCtx->cs.u32Limit == 0xffff);
6379	Assert(u32CSAttr == 0xf3);
6380	/* SS */
6381	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6382	Assert(pCtx->ss.u32Limit == 0xffff);
6383	Assert(u32SSAttr == 0xf3);
6384	/* DS */
6385	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6386	Assert(pCtx->ds.u32Limit == 0xffff);
6387	Assert(u32DSAttr == 0xf3);
6388	/* ES */
6389	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6390	Assert(pCtx->es.u32Limit == 0xffff);
6391	Assert(u32ESAttr == 0xf3);
6392	/* FS */
6393	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6394	Assert(pCtx->fs.u32Limit == 0xffff);
6395	Assert(u32FSAttr == 0xf3);
6396	/* GS */
6397	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6398	Assert(pCtx->gs.u32Limit == 0xffff);
6399	Assert(u32GSAttr == 0xf3);
6400	/* 64-bit capable CPUs. */
6401	# if HC_ARCH_BITS == 64
6402	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6403	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6404	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6405	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6406	# endif
6407	}
6408	}
6409	#endif /* VBOX_STRICT */
6410
6411
6412	/**
6413	* Exports a guest segment register into the guest-state area in the VMCS.
6414	*
6415	* @returns VBox status code.
6416	* @param pVCpu The cross context virtual CPU structure.
6417	* @param pVmcsInfo The VMCS info. object.
6418	* @param iSegReg The segment register number (X86_SREG_XXX).
6419	* @param pSelReg Pointer to the segment selector.
6420	*
6421	* @remarks No-long-jump zone!!!
6422	*/
6423	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6424	{
6425	Assert(iSegReg < X86_SREG_COUNT);
6426	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6427	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6428	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6429	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6430
6431	uint32_t u32Access = pSelReg->Attr.u;
6432	if (pVmcsInfo->RealMode.fRealOnV86Active)
6433	{
6434	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6435	u32Access = 0xf3;
6436	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6437	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6438	RT_NOREF_PV(pVCpu);
6439	}
6440	else
6441	{
6442	/*
6443	* The way to differentiate between whether this is really a null selector or was just
6444	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6445	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6446	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6447	* NULL selectors loaded in protected-mode have their attribute as 0.
6448	*/
6449	if (!u32Access)
6450	u32Access = X86DESCATTR_UNUSABLE;
6451	}
6452
6453	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6454	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6455	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6456
6457	/*
6458	* Commit it to the VMCS.
6459	*/
6460	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6461	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6462	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6463	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6464	AssertRCReturn(rc, rc);
6465	return rc;
6466	}
6467
6468
6469	/**
6470	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6471	* area in the VMCS.
6472	*
6473	* @returns VBox status code.
6474	* @param pVCpu The cross context virtual CPU structure.
6475	* @param pVmxTransient The VMX-transient structure.
6476	*
6477	* @remarks Will import guest CR0 on strict builds during validation of
6478	* segments.
6479	* @remarks No-long-jump zone!!!
6480	*/
6481	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6482	{
6483	int rc = VERR_INTERNAL_ERROR_5;
6484	PVM pVM = pVCpu->CTX_SUFF(pVM);
6485	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6486	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6487
6488	/*
6489	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6490	*/
6491	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6492	{
6493	#ifdef VBOX_WITH_REM
6494	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6495	{
6496	Assert(!pVmxTransient->fIsNestedGuest);
6497	Assert(pVM->hm.s.vmx.pRealModeTSS);
6498	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6499	if ( pVmcsInfo->fWasInRealMode
6500	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6501	{
6502	/*
6503	* Notify the recompiler must flush its code-cache as the guest -may-
6504	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6505	*/
6506	REMFlushTBs(pVM);
6507	Log4Func(("Switch to protected mode detected!\n"));
6508	pVmcsInfo->fWasInRealMode = false;
6509	}
6510	}
6511	#endif
6512	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6513	{
6514	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6515	if (pVmcsInfo->RealMode.fRealOnV86Active)
6516	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6517	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6518	AssertRCReturn(rc, rc);
6519	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6520	}
6521
6522	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6523	{
6524	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6525	if (pVmcsInfo->RealMode.fRealOnV86Active)
6526	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6527	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6528	AssertRCReturn(rc, rc);
6529	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6530	}
6531
6532	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6533	{
6534	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6535	if (pVmcsInfo->RealMode.fRealOnV86Active)
6536	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6537	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6538	AssertRCReturn(rc, rc);
6539	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6540	}
6541
6542	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6543	{
6544	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6545	if (pVmcsInfo->RealMode.fRealOnV86Active)
6546	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6547	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6548	AssertRCReturn(rc, rc);
6549	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6550	}
6551
6552	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6553	{
6554	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6555	if (pVmcsInfo->RealMode.fRealOnV86Active)
6556	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6557	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6558	AssertRCReturn(rc, rc);
6559	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6560	}
6561
6562	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6563	{
6564	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6565	if (pVmcsInfo->RealMode.fRealOnV86Active)
6566	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6567	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6568	AssertRCReturn(rc, rc);
6569	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6570	}
6571
6572	#ifdef VBOX_STRICT
6573	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6574	#endif
6575	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6576	pCtx->cs.Attr.u));
6577	}
6578
6579	/*
6580	* Guest TR.
6581	*/
6582	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6583	{
6584	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6585
6586	/*
6587	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6588	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6589	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6590	*/
6591	uint16_t u16Sel;
6592	uint32_t u32Limit;
6593	uint64_t u64Base;
6594	uint32_t u32AccessRights;
6595	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6596	{
6597	u16Sel = pCtx->tr.Sel;
6598	u32Limit = pCtx->tr.u32Limit;
6599	u64Base = pCtx->tr.u64Base;
6600	u32AccessRights = pCtx->tr.Attr.u;
6601	}
6602	else
6603	{
6604	Assert(!pVmxTransient->fIsNestedGuest);
6605	Assert(pVM->hm.s.vmx.pRealModeTSS);
6606	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6607
6608	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6609	RTGCPHYS GCPhys;
6610	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6611	AssertRCReturn(rc, rc);
6612
6613	X86DESCATTR DescAttr;
6614	DescAttr.u = 0;
6615	DescAttr.n.u1Present = 1;
6616	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6617
6618	u16Sel = 0;
6619	u32Limit = HM_VTX_TSS_SIZE;
6620	u64Base = GCPhys;
6621	u32AccessRights = DescAttr.u;
6622	}
6623
6624	/* Validate. */
6625	Assert(!(u16Sel & RT_BIT(2)));
6626	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6627	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6628	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6629	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6630	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6631	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6632	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6633	Assert( (u32Limit & 0xfff) == 0xfff
6634	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6635	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6636	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6637
6638	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6639	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6640	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6641	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6642	AssertRCReturn(rc, rc);
6643
6644	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6645	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6646	}
6647
6648	/*
6649	* Guest GDTR.
6650	*/
6651	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6652	{
6653	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6654
6655	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6656	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6657	AssertRCReturn(rc, rc);
6658
6659	/* Validate. */
6660	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6661
6662	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6663	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6664	}
6665
6666	/*
6667	* Guest LDTR.
6668	*/
6669	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6670	{
6671	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6672
6673	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6674	uint32_t u32Access;
6675	if ( !pVmxTransient->fIsNestedGuest
6676	&& !pCtx->ldtr.Attr.u)
6677	u32Access = X86DESCATTR_UNUSABLE;
6678	else
6679	u32Access = pCtx->ldtr.Attr.u;
6680
6681	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6682	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6683	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6684	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6685	AssertRCReturn(rc, rc);
6686
6687	/* Validate. */
6688	if (!(u32Access & X86DESCATTR_UNUSABLE))
6689	{
6690	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6691	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6692	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6693	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6694	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6695	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6696	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6697	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6698	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6699	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6700	}
6701
6702	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6703	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6704	}
6705
6706	/*
6707	* Guest IDTR.
6708	*/
6709	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6710	{
6711	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6712
6713	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6714	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6715	AssertRCReturn(rc, rc);
6716
6717	/* Validate. */
6718	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6719
6720	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6721	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6722	}
6723
6724	return VINF_SUCCESS;
6725	}
6726
6727
6728	/**
6729	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6730	* areas.
6731	*
6732	* These MSRs will automatically be loaded to the host CPU on every successful
6733	* VM-entry and stored from the host CPU on every successful VM-exit.
6734	*
6735	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6736	* actual host MSR values are not- updated here for performance reasons. See
6737	* hmR0VmxExportHostMsrs().
6738	*
6739	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6740	*
6741	* @returns VBox status code.
6742	* @param pVCpu The cross context virtual CPU structure.
6743	* @param pVmxTransient The VMX-transient structure.
6744	*
6745	* @remarks No-long-jump zone!!!
6746	*/
6747	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6748	{
6749	AssertPtr(pVCpu);
6750	AssertPtr(pVmxTransient);
6751
6752	PVM pVM = pVCpu->CTX_SUFF(pVM);
6753	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6754
6755	/*
6756	* MSRs that we use the auto-load/store MSR area in the VMCS.
6757	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6758	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6759	*
6760	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6761	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6762	* emulation, nothing to do here.
6763	*/
6764	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6765	{
6766	if ( !pVmxTransient->fIsNestedGuest
6767	&& pVM->hm.s.fAllow64BitGuests)
6768	{
6769	#if HC_ARCH_BITS == 32
6770	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6771	Assert(!pVmxTransient->fIsNestedGuest);
6772
6773	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6774	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6775	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6776	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6777	AssertRCReturn(rc, rc);
6778	#endif
6779	}
6780	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6781	}
6782
6783	/*
6784	* Guest Sysenter MSRs.
6785	*/
6786	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6787	{
6788	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6789
6790	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6791	{
6792	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6793	AssertRCReturn(rc, rc);
6794	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6795	}
6796
6797	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6798	{
6799	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6800	AssertRCReturn(rc, rc);
6801	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6802	}
6803
6804	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6805	{
6806	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6807	AssertRCReturn(rc, rc);
6808	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6809	}
6810	}
6811
6812	/*
6813	* Guest/host EFER MSR.
6814	*/
6815	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6816	{
6817	/* Whether we are using the VMCS to swap the EFER MSR must have been
6818	determined earlier while exporting VM-entry/VM-exit controls. */
6819	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6820	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6821
6822	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6823	{
6824	/*
6825	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6826	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6827	*/
6828	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6829	{
6830	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6831	AssertRCReturn(rc, rc);
6832	}
6833	else
6834	{
6835	/*
6836	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6837	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6838	*/
6839	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6840	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6841	AssertRCReturn(rc, rc);
6842	}
6843	}
6844	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6845	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6846
6847	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6848	}
6849
6850	/*
6851	* Other MSRs.
6852	* Speculation Control (R/W).
6853	*/
6854	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6855	{
6856	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6857	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6858	{
6859	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6860	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6861	AssertRCReturn(rc, rc);
6862	}
6863	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6864	}
6865
6866	return VINF_SUCCESS;
6867	}
6868
6869
6870	/**
6871	* Selects up the appropriate function to run guest code.
6872	*
6873	* @returns VBox status code.
6874	* @param pVCpu The cross context virtual CPU structure.
6875	* @param pVmxTransient The VMX-transient structure.
6876	*
6877	* @remarks No-long-jump zone!!!
6878	*/
6879	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6880	{
6881	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6882	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6883
6884	if (CPUMIsGuestInLongModeEx(pCtx))
6885	{
6886	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6887	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6888	#endif
6889	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6890	#if HC_ARCH_BITS == 32
6891	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6892	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6893	{
6894	#ifdef VBOX_STRICT
6895	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6896	{
6897	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6898	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6899	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6900	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6901	("fCtxChanged=%#RX64\n", fCtxChanged));
6902	}
6903	#endif
6904	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6905
6906	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6907	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6908	pVmcsInfo->fSwitchedTo64on32 = true;
6909	Log4Func(("Selected 64-bit switcher\n"));
6910	}
6911	#else
6912	/* 64-bit host. */
6913	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6914	#endif
6915	}
6916	else
6917	{
6918	/* Guest is not in long mode, use the 32-bit handler. */
6919	#if HC_ARCH_BITS == 32
6920	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6921	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6922	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6923	{
6924	# ifdef VBOX_STRICT
6925	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6926	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6927	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6928	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6929	("fCtxChanged=%#RX64\n", fCtxChanged));
6930	# endif
6931	}
6932	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6933	/*
6934	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6935	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6936	* switcher flag now because we know the guest is in a sane state where it's safe
6937	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6938	* the much faster 32-bit switcher again.
6939	*/
6940	if (!pVmcsInfo->fSwitchedTo64on32)
6941	{
6942	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6943	Log4Func(("Selected 32-bit switcher\n"));
6944	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6945	}
6946	else
6947	{
6948	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6949	if ( pVmcsInfo->RealMode.fRealOnV86Active
6950	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6951	{
6952	pVmcsInfo->fSwitchedTo64on32 = false;
6953	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6954	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6955	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6956	\| HM_CHANGED_HOST_CONTEXT);
6957	Log4Func(("Selected 32-bit switcher (safe)\n"));
6958	}
6959	}
6960	# else
6961	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6962	# endif
6963	#else
6964	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6965	#endif
6966	}
6967	Assert(pVmcsInfo->pfnStartVM);
6968	return VINF_SUCCESS;
6969	}
6970
6971
6972	/**
6973	* Wrapper for running the guest code in VT-x.
6974	*
6975	* @returns VBox status code, no informational status codes.
6976	* @param pVCpu The cross context virtual CPU structure.
6977	* @param pVmxTransient The VMX-transient structure.
6978	*
6979	* @remarks No-long-jump zone!!!
6980	*/
6981	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6982	{
6983	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6984	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6985	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6986
6987	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6988
6989	/*
6990	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6991	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6992	* callee-saved and thus the need for this XMM wrapper.
6993	*
6994	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6995	*/
6996	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6997	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6998	PVM pVM = pVCpu->CTX_SUFF(pVM);
6999	#ifdef VBOX_WITH_KERNEL_USING_XMM
7000	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
7001	#else
7002	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
7003	#endif
7004	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
7005	return rc;
7006	}
7007
7008
7009	/**
7010	* Reports world-switch error and dumps some useful debug info.
7011	*
7012	* @param pVCpu The cross context virtual CPU structure.
7013	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
7014	* @param pVmxTransient The VMX-transient structure (only
7015	* exitReason updated).
7016	*/
7017	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
7018	{
7019	Assert(pVCpu);
7020	Assert(pVmxTransient);
7021	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7022
7023	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
7024	switch (rcVMRun)
7025	{
7026	case VERR_VMX_INVALID_VMXON_PTR:
7027	AssertFailed();
7028	break;
7029	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
7030	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
7031	{
7032	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
7033	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
7034	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
7035	AssertRC(rc);
7036
7037	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7038	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
7039	Cannot do it here as we may have been long preempted. */
7040
7041	#ifdef VBOX_STRICT
7042	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7043	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
7044	pVmxTransient->uExitReason));
7045	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
7046	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
7047	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
7048	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
7049	else
7050	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
7051	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
7052	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
7053
7054	/* VMX control bits. */
7055	uint32_t u32Val;
7056	uint64_t u64Val;
7057	RTHCUINTREG uHCReg;
7058	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
7059	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
7060	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
7061	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
7062	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
7063	{
7064	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
7065	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
7066	}
7067	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
7068	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
7069	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
7070	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
7071	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
7072	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
7073	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
7074	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
7075	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
7076	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
7077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
7078	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
7079	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
7080	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
7081	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
7082	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
7083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7084	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
7085	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7086	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
7087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
7088	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
7089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
7090	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
7091	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
7092	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
7093	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7094	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7095	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7096	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7097	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7098	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7099	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7100	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7101	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7102	{
7103	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7104	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7105	}
7106
7107	/* Guest bits. */
7108	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
7109	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
7110	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
7111	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
7112	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
7113	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
7114	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
7115	{
7116	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
7117	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
7118	}
7119
7120	/* Host bits. */
7121	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
7122	Log4(("Host CR0 %#RHr\n", uHCReg));
7123	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
7124	Log4(("Host CR3 %#RHr\n", uHCReg));
7125	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
7126	Log4(("Host CR4 %#RHr\n", uHCReg));
7127
7128	RTGDTR HostGdtr;
7129	PCX86DESCHC pDesc;
7130	ASMGetGDTR(&HostGdtr);
7131	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
7132	Log4(("Host CS %#08x\n", u32Val));
7133	if (u32Val < HostGdtr.cbGdt)
7134	{
7135	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7136	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
7137	}
7138
7139	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
7140	Log4(("Host DS %#08x\n", u32Val));
7141	if (u32Val < HostGdtr.cbGdt)
7142	{
7143	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7144	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
7145	}
7146
7147	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
7148	Log4(("Host ES %#08x\n", u32Val));
7149	if (u32Val < HostGdtr.cbGdt)
7150	{
7151	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7152	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
7153	}
7154
7155	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
7156	Log4(("Host FS %#08x\n", u32Val));
7157	if (u32Val < HostGdtr.cbGdt)
7158	{
7159	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7160	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
7161	}
7162
7163	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
7164	Log4(("Host GS %#08x\n", u32Val));
7165	if (u32Val < HostGdtr.cbGdt)
7166	{
7167	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7168	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
7169	}
7170
7171	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
7172	Log4(("Host SS %#08x\n", u32Val));
7173	if (u32Val < HostGdtr.cbGdt)
7174	{
7175	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7176	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
7177	}
7178
7179	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
7180	Log4(("Host TR %#08x\n", u32Val));
7181	if (u32Val < HostGdtr.cbGdt)
7182	{
7183	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7184	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
7185	}
7186
7187	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
7188	Log4(("Host TR Base %#RHv\n", uHCReg));
7189	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
7190	Log4(("Host GDTR Base %#RHv\n", uHCReg));
7191	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
7192	Log4(("Host IDTR Base %#RHv\n", uHCReg));
7193	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
7194	Log4(("Host SYSENTER CS %#08x\n", u32Val));
7195	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
7196	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
7197	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
7198	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
7199	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
7200	Log4(("Host RSP %#RHv\n", uHCReg));
7201	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
7202	Log4(("Host RIP %#RHv\n", uHCReg));
7203	# if HC_ARCH_BITS == 64
7204	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
7205	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
7206	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
7207	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
7208	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
7209	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
7210	# endif
7211	#endif /* VBOX_STRICT */
7212	break;
7213	}
7214
7215	default:
7216	/* Impossible */
7217	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
7218	break;
7219	}
7220	}
7221
7222
7223	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
7224	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
7225	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
7226	# endif
7227
7228	/**
7229	* Initialize the VMCS-Read cache.
7230	*
7231	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
7232	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
7233	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
7234	* (those that have a 32-bit FULL & HIGH part).
7235	*
7236	* @param pVCpu The cross context virtual CPU structure.
7237	*/
7238	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
7239	{
7240	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
7241	do { \
7242	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
7243	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
7244	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
7245	++cReadFields; \
7246	} while (0)
7247
7248	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7249	uint32_t cReadFields = 0;
7250
7251	/*
7252	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
7253	* and serve to indicate exceptions to the rules.
7254	*/
7255
7256	/* Guest-natural selector base fields. */
7257	#if 0
7258	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
7259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
7260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
7261	#endif
7262	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
7263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
7264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
7265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
7266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
7267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
7268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
7269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
7270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
7271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
7272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
7273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
7274	#if 0
7275	/* Unused natural width guest-state fields. */
7276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
7277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
7278	#endif
7279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
7280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
7281
7282	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
7283	these 64-bit fields (using "FULL" and "HIGH" fields). */
7284	#if 0
7285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
7286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
7287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
7288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
7289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
7290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
7291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
7292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
7293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
7294	#endif
7295
7296	/* Natural width guest-state fields. */
7297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
7298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
7299
7300	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7301	{
7302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
7303	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
7304	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
7305	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
7306	}
7307	else
7308	{
7309	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
7310	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
7311	}
7312
7313	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
7314	}
7315
7316
7317	/**
7318	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
7319	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
7320	* darwin, running 64-bit guests).
7321	*
7322	* @returns VBox status code.
7323	* @param pVCpu The cross context virtual CPU structure.
7324	* @param idxField The VMCS field encoding.
7325	* @param u64Val 16, 32 or 64-bit value.
7326	*/
7327	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7328	{
7329	AssertPtr(pVCpu);
7330	int rc;
7331	switch (idxField)
7332	{
7333	/*
7334	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
7335	*/
7336	/* 64-bit Control fields. */
7337	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
7338	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
7339	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
7340	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
7341	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
7342	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
7343	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
7344	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
7345	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
7346	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
7347	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
7348	case VMX_VMCS64_CTRL_EPTP_FULL:
7349	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
7350	/* 64-bit Guest-state fields. */
7351	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
7352	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
7353	case VMX_VMCS64_GUEST_PAT_FULL:
7354	case VMX_VMCS64_GUEST_EFER_FULL:
7355	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
7356	case VMX_VMCS64_GUEST_PDPTE0_FULL:
7357	case VMX_VMCS64_GUEST_PDPTE1_FULL:
7358	case VMX_VMCS64_GUEST_PDPTE2_FULL:
7359	case VMX_VMCS64_GUEST_PDPTE3_FULL:
7360	/* 64-bit Host-state fields. */
7361	case VMX_VMCS64_HOST_PAT_FULL:
7362	case VMX_VMCS64_HOST_EFER_FULL:
7363	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
7364	{
7365	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7366	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
7367	break;
7368	}
7369
7370	/*
7371	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
7372	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
7373	*/
7374	/* Natural-width Guest-state fields. */
7375	case VMX_VMCS_GUEST_CR3:
7376	case VMX_VMCS_GUEST_ES_BASE:
7377	case VMX_VMCS_GUEST_CS_BASE:
7378	case VMX_VMCS_GUEST_SS_BASE:
7379	case VMX_VMCS_GUEST_DS_BASE:
7380	case VMX_VMCS_GUEST_FS_BASE:
7381	case VMX_VMCS_GUEST_GS_BASE:
7382	case VMX_VMCS_GUEST_LDTR_BASE:
7383	case VMX_VMCS_GUEST_TR_BASE:
7384	case VMX_VMCS_GUEST_GDTR_BASE:
7385	case VMX_VMCS_GUEST_IDTR_BASE:
7386	case VMX_VMCS_GUEST_RSP:
7387	case VMX_VMCS_GUEST_RIP:
7388	case VMX_VMCS_GUEST_SYSENTER_ESP:
7389	case VMX_VMCS_GUEST_SYSENTER_EIP:
7390	{
7391	if (!(RT_HI_U32(u64Val)))
7392	{
7393	/* If this field is 64-bit, VT-x will zero out the top bits. */
7394	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7395	}
7396	else
7397	{
7398	/* Assert that only the 32->64 switcher case should ever come here. */
7399	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
7400	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
7401	}
7402	break;
7403	}
7404
7405	default:
7406	{
7407	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
7408	pVCpu->hm.s.u32HMError = idxField;
7409	rc = VERR_INVALID_PARAMETER;
7410	break;
7411	}
7412	}
7413	AssertRCReturn(rc, rc);
7414	return rc;
7415	}
7416
7417
7418	/**
7419	* Queue up a VMWRITE by using the VMCS write cache.
7420	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
7421	*
7422	* @param pVCpu The cross context virtual CPU structure.
7423	* @param idxField The VMCS field encoding.
7424	* @param u64Val 16, 32 or 64-bit value.
7425	*/
7426	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7427	{
7428	AssertPtr(pVCpu);
7429	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7430
7431	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
7432	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
7433
7434	/* Make sure there are no duplicates. */
7435	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
7436	{
7437	if (pCache->Write.aField[i] == idxField)
7438	{
7439	pCache->Write.aFieldVal[i] = u64Val;
7440	return VINF_SUCCESS;
7441	}
7442	}
7443
7444	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
7445	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
7446	pCache->Write.cValidEntries++;
7447	return VINF_SUCCESS;
7448	}
7449	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
7450
7451
7452	/**
7453	* Sets up the usage of TSC-offsetting and updates the VMCS.
7454	*
7455	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
7456	* VMX-preemption timer.
7457	*
7458	* @returns VBox status code.
7459	* @param pVCpu The cross context virtual CPU structure.
7460	* @param pVmxTransient The VMX-transient structure.
7461	*
7462	* @remarks No-long-jump zone!!!
7463	*/
7464	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
7465	{
7466	bool fOffsettedTsc;
7467	bool fParavirtTsc;
7468	uint64_t uTscOffset;
7469	PVM pVM = pVCpu->CTX_SUFF(pVM);
7470	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7471
7472	if (pVM->hm.s.vmx.fUsePreemptTimer)
7473	{
7474	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
7475
7476	/* Make sure the returned values have sane upper and lower boundaries. */
7477	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
7478	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
7479	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
7480	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
7481
7482	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
7483	* preemption timers here. We probably need to clamp the preemption timer,
7484	* after converting the timer value to the host. */
7485	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7486	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7487	AssertRC(rc);
7488	}
7489	else
7490	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7491
7492	if (fParavirtTsc)
7493	{
7494	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7495	information before every VM-entry, hence disable it for performance sake. */
7496	#if 0
7497	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7498	AssertRC(rc);
7499	#endif
7500	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7501	}
7502
7503	if ( fOffsettedTsc
7504	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7505	{
7506	if (pVmxTransient->fIsNestedGuest)
7507	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7508	hmR0VmxSetTscOffsetVmcs(pVCpu, pVmcsInfo, uTscOffset);
7509	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7510	}
7511	else
7512	{
7513	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7514	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7515	}
7516	}
7517
7518
7519	/**
7520	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7521	* VM-exit interruption info type.
7522	*
7523	* @returns The IEM exception flags.
7524	* @param uVector The event vector.
7525	* @param uVmxEventType The VMX event type.
7526	*
7527	* @remarks This function currently only constructs flags required for
7528	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7529	* and CR2 aspects of an exception are not included).
7530	*/
7531	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7532	{
7533	uint32_t fIemXcptFlags;
7534	switch (uVmxEventType)
7535	{
7536	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7537	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7538	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7539	break;
7540
7541	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7542	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7543	break;
7544
7545	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7546	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7547	break;
7548
7549	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7550	{
7551	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7552	if (uVector == X86_XCPT_BP)
7553	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7554	else if (uVector == X86_XCPT_OF)
7555	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7556	else
7557	{
7558	fIemXcptFlags = 0;
7559	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7560	}
7561	break;
7562	}
7563
7564	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7565	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7566	break;
7567
7568	default:
7569	fIemXcptFlags = 0;
7570	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7571	break;
7572	}
7573	return fIemXcptFlags;
7574	}
7575
7576
7577	/**
7578	* Sets an event as a pending event to be injected into the guest.
7579	*
7580	* @param pVCpu The cross context virtual CPU structure.
7581	* @param u32IntInfo The VM-entry interruption-information field.
7582	* @param cbInstr The VM-entry instruction length in bytes (for software
7583	* interrupts, exceptions and privileged software
7584	* exceptions).
7585	* @param u32ErrCode The VM-entry exception error code.
7586	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7587	* page-fault.
7588	*/
7589	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7590	RTGCUINTPTR GCPtrFaultAddress)
7591	{
7592	Assert(!pVCpu->hm.s.Event.fPending);
7593	pVCpu->hm.s.Event.fPending = true;
7594	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7595	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7596	pVCpu->hm.s.Event.cbInstr = cbInstr;
7597	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7598	}
7599
7600
7601	/**
7602	* Sets an external interrupt as pending-for-injection into the VM.
7603	*
7604	* @param pVCpu The cross context virtual CPU structure.
7605	* @param u8Interrupt The external interrupt vector.
7606	*/
7607	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7608	{
7609	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7610	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7611	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7612	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7613	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7614	}
7615
7616
7617	/**
7618	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7619	*
7620	* @param pVCpu The cross context virtual CPU structure.
7621	*/
7622	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7623	{
7624	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7625	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7626	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7627	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7628	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7629	}
7630
7631
7632	/**
7633	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7634	*
7635	* @param pVCpu The cross context virtual CPU structure.
7636	*/
7637	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7638	{
7639	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7640	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7641	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7642	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7643	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7644	}
7645
7646
7647	/**
7648	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7649	*
7650	* @param pVCpu The cross context virtual CPU structure.
7651	*/
7652	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7653	{
7654	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7655	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7656	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7657	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7658	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7659	}
7660
7661
7662	/**
7663	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7664	*
7665	* @param pVCpu The cross context virtual CPU structure.
7666	*/
7667	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7668	{
7669	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7670	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7672	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7673	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7674	}
7675
7676
7677	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7678	/**
7679	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7680	*
7681	* @param pVCpu The cross context virtual CPU structure.
7682	* @param u32ErrCode The error code for the general-protection exception.
7683	*/
7684	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7685	{
7686	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7687	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7688	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7689	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7690	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7691	}
7692
7693
7694	/**
7695	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7696	*
7697	* @param pVCpu The cross context virtual CPU structure.
7698	* @param u32ErrCode The error code for the stack exception.
7699	*/
7700	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7701	{
7702	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7703	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7704	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7705	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7706	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7707	}
7708	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7709
7710
7711	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7712	{
7713	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7714
7715	/*
7716	* If VT-x marks the segment as unusable, most other bits remain undefined:
7717	* - For CS the L, D and G bits have meaning.
7718	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7719	* - For the remaining data segments no bits are defined.
7720	*
7721	* The present bit and the unusable bit has been observed to be set at the
7722	* same time (the selector was supposed to be invalid as we started executing
7723	* a V8086 interrupt in ring-0).
7724	*
7725	* What should be important for the rest of the VBox code, is that the P bit is
7726	* cleared. Some of the other VBox code recognizes the unusable bit, but
7727	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7728	* safe side here, we'll strip off P and other bits we don't care about. If
7729	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7730	*
7731	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7732	*/
7733	#ifdef VBOX_STRICT
7734	uint32_t const uAttr = pSelReg->Attr.u;
7735	#endif
7736
7737	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7738	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7739	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7740
7741	#ifdef VBOX_STRICT
7742	VMMRZCallRing3Disable(pVCpu);
7743	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7744	# ifdef DEBUG_bird
7745	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7746	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7747	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7748	# endif
7749	VMMRZCallRing3Enable(pVCpu);
7750	NOREF(uAttr);
7751	#endif
7752	RT_NOREF2(pVCpu, idxSel);
7753	}
7754
7755
7756	/**
7757	* Imports a guest segment register from the current VMCS into the guest-CPU
7758	* context.
7759	*
7760	* @returns VBox status code.
7761	* @param pVCpu The cross context virtual CPU structure.
7762	* @param iSegReg The segment register number (X86_SREG_XXX).
7763	*
7764	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7765	* do not log!
7766	*/
7767	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7768	{
7769	Assert(iSegReg < X86_SREG_COUNT);
7770
7771	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7772	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7773	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7774	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7775	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7776	#else
7777	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7778	#endif
7779	uint64_t u64Base;
7780	uint32_t u32Sel, u32Limit, u32Attr;
7781	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7782	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7783	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7784	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7785	if (RT_SUCCESS(rc))
7786	{
7787	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7788	pSelReg->Sel = u32Sel;
7789	pSelReg->ValidSel = u32Sel;
7790	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7791	pSelReg->u32Limit = u32Limit;
7792	pSelReg->u64Base = u64Base;
7793	pSelReg->Attr.u = u32Attr;
7794	if (u32Attr & X86DESCATTR_UNUSABLE)
7795	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7796	}
7797	return rc;
7798	}
7799
7800
7801	/**
7802	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7803	*
7804	* @returns VBox status code.
7805	* @param pVCpu The cross context virtual CPU structure.
7806	*
7807	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7808	* do not log!
7809	*/
7810	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7811	{
7812	uint64_t u64Base;
7813	uint32_t u32Sel, u32Limit, u32Attr;
7814	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7815	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7816	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7817	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7818	if (RT_SUCCESS(rc))
7819	{
7820	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7821	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7822	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7823	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7824	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7825	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7826	if (u32Attr & X86DESCATTR_UNUSABLE)
7827	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7828	}
7829	return rc;
7830	}
7831
7832
7833	/**
7834	* Imports the guest TR from the current VMCS into the guest-CPU context.
7835	*
7836	* @returns VBox status code.
7837	* @param pVCpu The cross context virtual CPU structure.
7838	*
7839	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7840	* do not log!
7841	*/
7842	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7843	{
7844	uint32_t u32Sel, u32Limit, u32Attr;
7845	uint64_t u64Base;
7846	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7847	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7848	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7849	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7850	AssertRCReturn(rc, rc);
7851
7852	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7853	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7854	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7855	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7856	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7857	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7858	/* TR is the only selector that can never be unusable. */
7859	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7860	return VINF_SUCCESS;
7861	}
7862
7863
7864	/**
7865	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7866	*
7867	* @returns VBox status code.
7868	* @param pVCpu The cross context virtual CPU structure.
7869	*
7870	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7871	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7872	* instead!!!
7873	*/
7874	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7875	{
7876	uint64_t u64Val;
7877	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7878	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7879	{
7880	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7881	if (RT_SUCCESS(rc))
7882	{
7883	pCtx->rip = u64Val;
7884	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7885	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7886	}
7887	return rc;
7888	}
7889	return VINF_SUCCESS;
7890	}
7891
7892
7893	/**
7894	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7895	*
7896	* @returns VBox status code.
7897	* @param pVCpu The cross context virtual CPU structure.
7898	* @param pVmcsInfo The VMCS info. object.
7899	*
7900	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7901	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7902	* instead!!!
7903	*/
7904	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7905	{
7906	uint32_t u32Val;
7907	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7908	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7909	{
7910	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7911	if (RT_SUCCESS(rc))
7912	{
7913	pCtx->eflags.u32 = u32Val;
7914
7915	/* Restore eflags for real-on-v86-mode hack. */
7916	if (pVmcsInfo->RealMode.fRealOnV86Active)
7917	{
7918	pCtx->eflags.Bits.u1VM = 0;
7919	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7920	}
7921	}
7922	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7923	return rc;
7924	}
7925	return VINF_SUCCESS;
7926	}
7927
7928
7929	/**
7930	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7931	* context.
7932	*
7933	* @returns VBox status code.
7934	* @param pVCpu The cross context virtual CPU structure.
7935	* @param pVmcsInfo The VMCS info. object.
7936	*
7937	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7938	* do not log!
7939	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7940	* instead!!!
7941	*/
7942	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7943	{
7944	uint32_t u32Val;
7945	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7946	if (RT_SUCCESS(rc))
7947	{
7948	if (!u32Val)
7949	{
7950	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7951	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7952
7953	CPUMSetGuestNmiBlocking(pVCpu, false);
7954	}
7955	else
7956	{
7957	/*
7958	* We must import RIP here to set our EM interrupt-inhibited state.
7959	* We also import RFLAGS as our code that evaluates pending interrupts
7960	* before VM-entry requires it.
7961	*/
7962	rc = hmR0VmxImportGuestRip(pVCpu);
7963	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7964	if (RT_SUCCESS(rc))
7965	{
7966	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7967	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7968	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7969	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7970
7971	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7972	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7973	}
7974	}
7975	}
7976	return rc;
7977	}
7978
7979
7980	/**
7981	* Worker for VMXR0ImportStateOnDemand.
7982	*
7983	* @returns VBox status code.
7984	* @param pVCpu The cross context virtual CPU structure.
7985	* @param pVmcsInfo The VMCS info. object.
7986	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7987	*/
7988	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7989	{
7990	#define VMXLOCAL_BREAK_RC(a_rc) \
7991	if (RT_SUCCESS(a_rc)) \
7992	{ } \
7993	else \
7994	break
7995
7996	int rc = VINF_SUCCESS;
7997	PVM pVM = pVCpu->CTX_SUFF(pVM);
7998	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7999	uint64_t u64Val;
8000	uint32_t u32Val;
8001
8002	/*
8003	* Note! This is hack to workaround a mysterious BSOD observed with release builds
8004	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
8005	* neither are other host platforms.
8006	*
8007	* Committing this temporarily as it prevents BSOD.
8008	*
8009	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
8010	*/
8011	#ifdef RT_OS_WINDOWS
8012	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
8013	return VERR_HM_IPE_1;
8014	#endif
8015
8016	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
8017
8018	/*
8019	* We disable interrupts to make the updating of the state and in particular
8020	* the fExtrn modification atomic wrt to preemption hooks.
8021	*/
8022	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
8023
8024	fWhat &= pCtx->fExtrn;
8025	if (fWhat)
8026	{
8027	do
8028	{
8029	if (fWhat & CPUMCTX_EXTRN_RIP)
8030	{
8031	rc = hmR0VmxImportGuestRip(pVCpu);
8032	VMXLOCAL_BREAK_RC(rc);
8033	}
8034
8035	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
8036	{
8037	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8038	VMXLOCAL_BREAK_RC(rc);
8039	}
8040
8041	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
8042	{
8043	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
8044	VMXLOCAL_BREAK_RC(rc);
8045	}
8046
8047	if (fWhat & CPUMCTX_EXTRN_RSP)
8048	{
8049	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
8050	VMXLOCAL_BREAK_RC(rc);
8051	pCtx->rsp = u64Val;
8052	}
8053
8054	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
8055	{
8056	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
8057	if (fWhat & CPUMCTX_EXTRN_CS)
8058	{
8059	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
8060	rc \|= hmR0VmxImportGuestRip(pVCpu);
8061	if (fRealOnV86Active)
8062	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
8063	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
8064	}
8065	if (fWhat & CPUMCTX_EXTRN_SS)
8066	{
8067	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
8068	if (fRealOnV86Active)
8069	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
8070	}
8071	if (fWhat & CPUMCTX_EXTRN_DS)
8072	{
8073	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
8074	if (fRealOnV86Active)
8075	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
8076	}
8077	if (fWhat & CPUMCTX_EXTRN_ES)
8078	{
8079	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
8080	if (fRealOnV86Active)
8081	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
8082	}
8083	if (fWhat & CPUMCTX_EXTRN_FS)
8084	{
8085	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
8086	if (fRealOnV86Active)
8087	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
8088	}
8089	if (fWhat & CPUMCTX_EXTRN_GS)
8090	{
8091	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
8092	if (fRealOnV86Active)
8093	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
8094	}
8095	VMXLOCAL_BREAK_RC(rc);
8096	}
8097
8098	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
8099	{
8100	if (fWhat & CPUMCTX_EXTRN_LDTR)
8101	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
8102
8103	if (fWhat & CPUMCTX_EXTRN_GDTR)
8104	{
8105	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8106	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8107	pCtx->gdtr.pGdt = u64Val;
8108	pCtx->gdtr.cbGdt = u32Val;
8109	}
8110
8111	/* Guest IDTR. */
8112	if (fWhat & CPUMCTX_EXTRN_IDTR)
8113	{
8114	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8115	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8116	pCtx->idtr.pIdt = u64Val;
8117	pCtx->idtr.cbIdt = u32Val;
8118	}
8119
8120	/* Guest TR. */
8121	if (fWhat & CPUMCTX_EXTRN_TR)
8122	{
8123	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
8124	don't need to import that one. */
8125	if (!pVmcsInfo->RealMode.fRealOnV86Active)
8126	rc \|= hmR0VmxImportGuestTr(pVCpu);
8127	}
8128	VMXLOCAL_BREAK_RC(rc);
8129	}
8130
8131	if (fWhat & CPUMCTX_EXTRN_DR7)
8132	{
8133	if (!pVCpu->hm.s.fUsingHyperDR7)
8134	{
8135	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
8136	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
8137	VMXLOCAL_BREAK_RC(rc);
8138	pCtx->dr[7] = u32Val;
8139	}
8140	}
8141
8142	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
8143	{
8144	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
8145	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
8146	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
8147	pCtx->SysEnter.cs = u32Val;
8148	VMXLOCAL_BREAK_RC(rc);
8149	}
8150
8151	#if HC_ARCH_BITS == 64
8152	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
8153	{
8154	if ( pVM->hm.s.fAllow64BitGuests
8155	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8156	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
8157	}
8158
8159	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
8160	{
8161	if ( pVM->hm.s.fAllow64BitGuests
8162	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8163	{
8164	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
8165	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
8166	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
8167	}
8168	}
8169	#endif
8170
8171	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
8172	#if HC_ARCH_BITS == 32
8173	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
8174	#endif
8175	)
8176	{
8177	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
8178	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
8179	Assert(pMsrs);
8180	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
8181	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
8182	for (uint32_t i = 0; i < cMsrs; i++)
8183	{
8184	uint32_t const idMsr = pMsrs[i].u32Msr;
8185	switch (idMsr)
8186	{
8187	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
8188	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
8189	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
8190	#if HC_ARCH_BITS == 32
8191	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
8192	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
8193	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
8194	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
8195	#endif
8196	default:
8197	{
8198	pCtx->fExtrn = 0;
8199	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
8200	ASMSetFlags(fEFlags);
8201	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
8202	return VERR_HM_UNEXPECTED_LD_ST_MSR;
8203	}
8204	}
8205	}
8206	}
8207
8208	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
8209	{
8210	uint64_t u64Shadow;
8211	if (fWhat & CPUMCTX_EXTRN_CR0)
8212	{
8213	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8214	* remove when we drop 32-bit host w/ 64-bit host support, see
8215	* @bugref{9180#c39}. */
8216	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8217	#if HC_ARCH_BITS == 32
8218	uint32_t u32Shadow;
8219	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
8220	u64Shadow = u32Shadow;
8221	#else
8222	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
8223	#endif
8224	VMXLOCAL_BREAK_RC(rc);
8225	u64Val = u32Val;
8226	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
8227	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
8228	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8229	/*
8230	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
8231	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
8232	*/
8233	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8234	{
8235	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
8236	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
8237	}
8238	#endif
8239	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
8240	CPUMSetGuestCR0(pVCpu, u64Val);
8241	VMMRZCallRing3Enable(pVCpu);
8242	}
8243
8244	if (fWhat & CPUMCTX_EXTRN_CR4)
8245	{
8246	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8247	* remove when we drop 32-bit host w/ 64-bit host support, see
8248	* @bugref{9180#c39}. */
8249	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8250	#if HC_ARCH_BITS == 32
8251	uint32_t u32Shadow;
8252	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
8253	u64Shadow = u32Shadow;
8254	#else
8255	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
8256	#endif
8257	VMXLOCAL_BREAK_RC(rc);
8258	u64Val = u32Val;
8259	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
8260	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
8261	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8262	/*
8263	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
8264	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
8265	*/
8266	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8267	{
8268	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
8269	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
8270	}
8271	#endif
8272	pCtx->cr4 = u64Val;
8273	}
8274
8275	if (fWhat & CPUMCTX_EXTRN_CR3)
8276	{
8277	/* CR0.PG bit changes are always intercepted, so it's up to date. */
8278	if ( pVM->hm.s.vmx.fUnrestrictedGuest
8279	\|\| ( pVM->hm.s.fNestedPaging
8280	&& CPUMIsGuestPagingEnabledEx(pCtx)))
8281	{
8282	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
8283	VMXLOCAL_BREAK_RC(rc);
8284	if (pCtx->cr3 != u64Val)
8285	{
8286	pCtx->cr3 = u64Val;
8287	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
8288	}
8289
8290	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
8291	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
8292	if (CPUMIsGuestInPAEModeEx(pCtx))
8293	{
8294	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
8295	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
8296	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
8297	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
8298	VMXLOCAL_BREAK_RC(rc);
8299	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
8300	}
8301	}
8302	}
8303	}
8304
8305	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8306	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
8307	{
8308	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
8309	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
8310	{
8311	Assert(CPUMIsGuestInVmxRootMode(pCtx));
8312	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
8313	VMXLOCAL_BREAK_RC(rc);
8314	}
8315
8316	# if 0
8317	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
8318	* VM-exit handlers as parameters. We would handle it differently when using
8319	* the fast path. */
8320	/*
8321	* The hardware virtualization state currently consists of VMCS fields that may be
8322	* modified by execution of the nested-guest (that are not part of the general
8323	* guest state) and is visible to guest software. Hence, it is technically part of
8324	* the guest-CPU state when executing a nested-guest.
8325	*/
8326	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8327	{
8328	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
8329	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
8330	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
8331	VMXLOCAL_BREAK_RC(rc);
8332
8333	/*
8334	* VM-entry can fail due to invalid-guest state, machine-check events and
8335	* MSR loading failures. Other than VM-exit reason and Exit qualification
8336	* all other VMCS fields are left unmodified on VM-entry failure.
8337	*
8338	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
8339	*/
8340	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
8341	if (!fEntryFailed)
8342	{
8343	/*
8344	* Some notes on VMCS fields that may need importing when the fast path
8345	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
8346	*
8347	* Requires fixing up when using hardware-assisted VMX:
8348	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
8349	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
8350	* - IDT-vectoring info: Think about this.
8351	* - IDT-vectoring error code: Think about this.
8352	*
8353	* Emulated:
8354	* - Guest-interruptiblity state: Derived from FFs and RIP.
8355	* - Guest pending debug exceptions: Derived from DR6.
8356	* - Guest activity state: Emulated from EM state.
8357	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
8358	* - Entry-interrupt info: Emulated, cleared to 0.
8359	*/
8360	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
8361	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
8362	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
8363	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
8364	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
8365	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
8366	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
8367	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
8368	/** @todo NSTVMX: Save and adjust preemption timer value. */
8369	}
8370
8371	VMXLOCAL_BREAK_RC(rc);
8372	}
8373	# endif
8374	}
8375	#endif
8376	} while (0);
8377
8378	if (RT_SUCCESS(rc))
8379	{
8380	/* Update fExtrn. */
8381	pCtx->fExtrn &= ~fWhat;
8382
8383	/* If everything has been imported, clear the HM keeper bit. */
8384	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
8385	{
8386	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
8387	Assert(!pCtx->fExtrn);
8388	}
8389	}
8390	}
8391	else
8392	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
8393
8394	ASMSetFlags(fEFlags);
8395
8396	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
8397
8398	if (RT_SUCCESS(rc))
8399	{ /* likely */ }
8400	else
8401	return rc;
8402
8403	/*
8404	* Honor any pending CR3 updates.
8405	*
8406	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
8407	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
8408	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
8409	*
8410	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
8411	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
8412	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
8413	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
8414	*
8415	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
8416	*/
8417	if (VMMRZCallRing3IsEnabled(pVCpu))
8418	{
8419	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8420	{
8421	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8422	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8423	}
8424
8425	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8426	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8427
8428	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8429	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8430	}
8431
8432	return VINF_SUCCESS;
8433	#undef VMXLOCAL_BREAK_RC
8434	}
8435
8436
8437	/**
8438	* Saves the guest state from the VMCS into the guest-CPU context.
8439	*
8440	* @returns VBox status code.
8441	* @param pVCpu The cross context virtual CPU structure.
8442	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8443	*/
8444	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
8445	{
8446	AssertPtr(pVCpu);
8447	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8448	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
8449	}
8450
8451
8452	/**
8453	* Check per-VM and per-VCPU force flag actions that require us to go back to
8454	* ring-3 for one reason or another.
8455	*
8456	* @returns Strict VBox status code (i.e. informational status codes too)
8457	* @retval VINF_SUCCESS if we don't have any actions that require going back to
8458	* ring-3.
8459	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
8460	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
8461	* interrupts)
8462	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
8463	* all EMTs to be in ring-3.
8464	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
8465	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
8466	* to the EM loop.
8467	*
8468	* @param pVCpu The cross context virtual CPU structure.
8469	* @param fStepping Whether we are single-stepping the guest using the
8470	* hypervisor debugger.
8471	*/
8472	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
8473	{
8474	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8475
8476	/*
8477	* Update pending interrupts into the APIC's IRR.
8478	*/
8479	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8480	APICUpdatePendingInterrupts(pVCpu);
8481
8482	/*
8483	* Anything pending? Should be more likely than not if we're doing a good job.
8484	*/
8485	PVM pVM = pVCpu->CTX_SUFF(pVM);
8486	if ( !fStepping
8487	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
8488	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
8489	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
8490	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
8491	return VINF_SUCCESS;
8492
8493	/* Pending PGM C3 sync. */
8494	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
8495	{
8496	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8497	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
8498	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8499	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8500	if (rcStrict2 != VINF_SUCCESS)
8501	{
8502	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8503	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8504	return rcStrict2;
8505	}
8506	}
8507
8508	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8509	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8510	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8511	{
8512	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8513	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8514	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8515	return rc2;
8516	}
8517
8518	/* Pending VM request packets, such as hardware interrupts. */
8519	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8520	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8521	{
8522	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
8523	Log4Func(("Pending VM request forcing us back to ring-3\n"));
8524	return VINF_EM_PENDING_REQUEST;
8525	}
8526
8527	/* Pending PGM pool flushes. */
8528	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8529	{
8530	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
8531	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8532	return VINF_PGM_POOL_FLUSH_PENDING;
8533	}
8534
8535	/* Pending DMA requests. */
8536	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8537	{
8538	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
8539	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8540	return VINF_EM_RAW_TO_R3;
8541	}
8542
8543	return VINF_SUCCESS;
8544	}
8545
8546
8547	/**
8548	* Converts any TRPM trap into a pending HM event. This is typically used when
8549	* entering from ring-3 (not longjmp returns).
8550	*
8551	* @param pVCpu The cross context virtual CPU structure.
8552	*/
8553	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8554	{
8555	Assert(TRPMHasTrap(pVCpu));
8556	Assert(!pVCpu->hm.s.Event.fPending);
8557
8558	uint8_t uVector;
8559	TRPMEVENT enmTrpmEvent;
8560	RTGCUINT uErrCode;
8561	RTGCUINTPTR GCPtrFaultAddress;
8562	uint8_t cbInstr;
8563
8564	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8565	AssertRC(rc);
8566
8567	uint32_t u32IntInfo;
8568	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
8569	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent);
8570
8571	rc = TRPMResetTrap(pVCpu);
8572	AssertRC(rc);
8573	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8574	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8575
8576	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8577	}
8578
8579
8580	/**
8581	* Converts the pending HM event into a TRPM trap.
8582	*
8583	* @param pVCpu The cross context virtual CPU structure.
8584	*/
8585	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8586	{
8587	Assert(pVCpu->hm.s.Event.fPending);
8588
8589	/* If a trap was already pending, we did something wrong! */
8590	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8591
8592	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
8593	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
8594	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
8595
8596	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8597
8598	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8599	AssertRC(rc);
8600
8601	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8602	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
8603
8604	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
8605	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8606	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
8607	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8608
8609	/* We're now done converting the pending event. */
8610	pVCpu->hm.s.Event.fPending = false;
8611	}
8612
8613
8614	/**
8615	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8616	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8617	*
8618	* @param pVCpu The cross context virtual CPU structure.
8619	* @param pVmcsInfo The VMCS info. object.
8620	*/
8621	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8622	{
8623	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8624	{
8625	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8626	{
8627	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8628	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8629	AssertRC(rc);
8630	}
8631	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8632	}
8633
8634
8635	/**
8636	* Clears the interrupt-window exiting control in the VMCS.
8637	*
8638	* @param pVmcsInfo The VMCS info. object.
8639	*/
8640	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8641	{
8642	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8643	{
8644	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8645	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8646	}
8647	return VINF_SUCCESS;
8648	}
8649
8650
8651	/**
8652	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8653	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8654	*
8655	* @param pVCpu The cross context virtual CPU structure.
8656	* @param pVmcsInfo The VMCS info. object.
8657	*/
8658	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8659	{
8660	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8661	{
8662	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8663	{
8664	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8665	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8666	AssertRC(rc);
8667	Log4Func(("Setup NMI-window exiting\n"));
8668	}
8669	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8670	}
8671
8672
8673	/**
8674	* Clears the NMI-window exiting control in the VMCS.
8675	*
8676	* @param pVmcsInfo The VMCS info. object.
8677	*/
8678	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8679	{
8680	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8681	{
8682	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8683	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8684	}
8685	return VINF_SUCCESS;
8686	}
8687
8688
8689	/**
8690	* Does the necessary state syncing before returning to ring-3 for any reason
8691	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8692	*
8693	* @returns VBox status code.
8694	* @param pVCpu The cross context virtual CPU structure.
8695	* @param fImportState Whether to import the guest state from the VMCS back
8696	* to the guest-CPU context.
8697	*
8698	* @remarks No-long-jmp zone!!!
8699	*/
8700	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8701	{
8702	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8703	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8704
8705	RTCPUID const idCpu = RTMpCpuId();
8706	Log4Func(("HostCpuId=%u\n", idCpu));
8707
8708	/*
8709	* !!! IMPORTANT !!!
8710	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8711	*/
8712
8713	/* Save the guest state if necessary. */
8714	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8715	if (fImportState)
8716	{
8717	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8718	AssertRCReturn(rc, rc);
8719	}
8720
8721	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8722	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8723	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8724
8725	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8726	#ifdef VBOX_STRICT
8727	if (CPUMIsHyperDebugStateActive(pVCpu))
8728	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8729	#endif
8730	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8731	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8732	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8733
8734	#if HC_ARCH_BITS == 64
8735	/* Restore host-state bits that VT-x only restores partially. */
8736	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8737	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8738	{
8739	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8740	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8741	}
8742	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8743	#endif
8744
8745	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8746	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8747	{
8748	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8749	if (!fImportState)
8750	{
8751	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8752	AssertRCReturn(rc, rc);
8753	}
8754	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8755	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8756	}
8757	else
8758	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8759
8760	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8761	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8762
8763	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8764	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8765	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8766	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8767	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8768	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8769	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8770	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8771	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8772	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8773
8774	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8775
8776	/** @todo This partially defeats the purpose of having preemption hooks.
8777	* The problem is, deregistering the hooks should be moved to a place that
8778	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8779	* context.
8780	*/
8781	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8782	AssertRCReturn(rc, rc);
8783
8784	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8785	/*
8786	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8787	* clear a shadow VMCS before allowing that VMCS to become active on another
8788	* logical processor. We may or may not be importing guest state which clears
8789	* it, so cover for it here.
8790	*
8791	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8792	*/
8793	if ( pVmcsInfo->pvShadowVmcs
8794	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8795	{
8796	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8797	AssertRCReturn(rc, rc);
8798	}
8799
8800	/*
8801	* Flag that we need to re-import the host state if we switch to this VMCS before
8802	* executing guest or nested-guest code.
8803	*/
8804	pVmcsInfo->idHostCpu = NIL_RTCPUID;
8805	#endif
8806
8807	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8808	NOREF(idCpu);
8809	return VINF_SUCCESS;
8810	}
8811
8812
8813	/**
8814	* Leaves the VT-x session.
8815	*
8816	* @returns VBox status code.
8817	* @param pVCpu The cross context virtual CPU structure.
8818	*
8819	* @remarks No-long-jmp zone!!!
8820	*/
8821	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8822	{
8823	HM_DISABLE_PREEMPT(pVCpu);
8824	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8825	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8826	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8827
8828	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8829	and done this from the VMXR0ThreadCtxCallback(). */
8830	if (!pVCpu->hm.s.fLeaveDone)
8831	{
8832	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8833	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8834	pVCpu->hm.s.fLeaveDone = true;
8835	}
8836	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8837
8838	/*
8839	* !!! IMPORTANT !!!
8840	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8841	*/
8842
8843	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8844	/** @todo Deregistering here means we need to VMCLEAR always
8845	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8846	* for calling VMMR0ThreadCtxHookDisable here! */
8847	VMMR0ThreadCtxHookDisable(pVCpu);
8848
8849	/* Leave HM context. This takes care of local init (term). */
8850	int rc = HMR0LeaveCpu(pVCpu);
8851
8852	HM_RESTORE_PREEMPT();
8853	return rc;
8854	}
8855
8856
8857	/**
8858	* Does the necessary state syncing before doing a longjmp to ring-3.
8859	*
8860	* @returns VBox status code.
8861	* @param pVCpu The cross context virtual CPU structure.
8862	*
8863	* @remarks No-long-jmp zone!!!
8864	*/
8865	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8866	{
8867	return hmR0VmxLeaveSession(pVCpu);
8868	}
8869
8870
8871	/**
8872	* Take necessary actions before going back to ring-3.
8873	*
8874	* An action requires us to go back to ring-3. This function does the necessary
8875	* steps before we can safely return to ring-3. This is not the same as longjmps
8876	* to ring-3, this is voluntary and prepares the guest so it may continue
8877	* executing outside HM (recompiler/IEM).
8878	*
8879	* @returns VBox status code.
8880	* @param pVCpu The cross context virtual CPU structure.
8881	* @param rcExit The reason for exiting to ring-3. Can be
8882	* VINF_VMM_UNKNOWN_RING3_CALL.
8883	*/
8884	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8885	{
8886	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8887
8888	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8889	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8890	{
8891	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8892	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8893	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8894	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8895	}
8896
8897	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8898	VMMRZCallRing3Disable(pVCpu);
8899	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8900
8901	/*
8902	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8903	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8904	*
8905	* This is because execution may continue from ring-3 and we would need to inject
8906	* the event from there (hence place it back in TRPM).
8907	*/
8908	if (pVCpu->hm.s.Event.fPending)
8909	{
8910	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8911	Assert(!pVCpu->hm.s.Event.fPending);
8912
8913	/* Clear the events from the VMCS. */
8914	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8915	AssertRCReturn(rc, rc);
8916	}
8917	#ifdef VBOX_STRICT
8918	else
8919	{
8920	/*
8921	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8922	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8923	* occasionally, see @bugref{9180#c42}.
8924	*
8925	* However, if the VM-entry failed, any VM entry-interruption info. field would
8926	* be left unmodified as the event would not have been injected to the guest. In
8927	* such cases, don't assert, we're not going to continue guest execution anyway.
8928	*/
8929	uint32_t uExitReason;
8930	uint32_t uEntryIntInfo;
8931	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8932	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8933	AssertRC(rc);
8934	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8935	}
8936	#endif
8937
8938	/*
8939	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8940	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8941	* (e.g. TPR below threshold).
8942	*/
8943	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8944	{
8945	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8946	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8947	AssertRCReturn(rc, rc);
8948	}
8949
8950	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8951	and if we're injecting an event we should have a TRPM trap pending. */
8952	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8953	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8954	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8955	#endif
8956
8957	/* Save guest state and restore host state bits. */
8958	int rc = hmR0VmxLeaveSession(pVCpu);
8959	AssertRCReturn(rc, rc);
8960	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8961
8962	/* Thread-context hooks are unregistered at this point!!! */
8963
8964	/* Sync recompiler state. */
8965	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8966	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8967	\| CPUM_CHANGED_LDTR
8968	\| CPUM_CHANGED_GDTR
8969	\| CPUM_CHANGED_IDTR
8970	\| CPUM_CHANGED_TR
8971	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8972	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8973	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8974	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8975
8976	Assert(!pVCpu->hm.s.fClearTrapFlag);
8977
8978	/* Update the exit-to-ring 3 reason. */
8979	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8980
8981	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8982	if ( rcExit != VINF_EM_RAW_INTERRUPT
8983	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8984	{
8985	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8986	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8987	}
8988
8989	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8990
8991	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8992	VMMRZCallRing3RemoveNotification(pVCpu);
8993	VMMRZCallRing3Enable(pVCpu);
8994
8995	return rc;
8996	}
8997
8998
8999	/**
9000	* VMMRZCallRing3() callback wrapper which saves the guest state before we
9001	* longjump to ring-3 and possibly get preempted.
9002	*
9003	* @returns VBox status code.
9004	* @param pVCpu The cross context virtual CPU structure.
9005	* @param enmOperation The operation causing the ring-3 longjump.
9006	* @param pvUser User argument, currently unused, NULL.
9007	*/
9008	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
9009	{
9010	RT_NOREF(pvUser);
9011	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
9012	{
9013	/*
9014	* !!! IMPORTANT !!!
9015	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
9016	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
9017	*/
9018	VMMRZCallRing3RemoveNotification(pVCpu);
9019	VMMRZCallRing3Disable(pVCpu);
9020	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
9021	RTThreadPreemptDisable(&PreemptState);
9022
9023	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9024	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9025	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
9026	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
9027
9028	#if HC_ARCH_BITS == 64
9029	/* Restore host-state bits that VT-x only restores partially. */
9030	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
9031	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
9032	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
9033	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
9034	#endif
9035
9036	/* Restore the lazy host MSRs as we're leaving VT-x context. */
9037	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
9038	hmR0VmxLazyRestoreHostMsrs(pVCpu);
9039
9040	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
9041	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
9042	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
9043
9044	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
9045	cleared as part of importing the guest state above. */
9046	hmR0VmxClearVmcs(pVmcsInfo);
9047
9048	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
9049	VMMR0ThreadCtxHookDisable(pVCpu);
9050	HMR0LeaveCpu(pVCpu);
9051	RTThreadPreemptRestore(&PreemptState);
9052	return VINF_SUCCESS;
9053	}
9054
9055	Assert(pVCpu);
9056	Assert(pvUser);
9057	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9058	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9059
9060	VMMRZCallRing3Disable(pVCpu);
9061	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9062
9063	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
9064
9065	int rc = hmR0VmxLongJmpToRing3(pVCpu);
9066	AssertRCReturn(rc, rc);
9067
9068	VMMRZCallRing3Enable(pVCpu);
9069	return VINF_SUCCESS;
9070	}
9071
9072
9073	/**
9074	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
9075	* stack.
9076	*
9077	* @returns Strict VBox status code (i.e. informational status codes too).
9078	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
9079	* @param pVCpu The cross context virtual CPU structure.
9080	* @param uValue The value to push to the guest stack.
9081	*/
9082	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
9083	{
9084	/*
9085	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
9086	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
9087	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
9088	*/
9089	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9090	if (pCtx->sp == 1)
9091	return VINF_EM_RESET;
9092	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
9093	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
9094	AssertRC(rc);
9095	return rc;
9096	}
9097
9098
9099	/**
9100	* Injects an event into the guest upon VM-entry by updating the relevant fields
9101	* in the VM-entry area in the VMCS.
9102	*
9103	* @returns Strict VBox status code (i.e. informational status codes too).
9104	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
9105	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
9106	*
9107	* @param pVCpu The cross context virtual CPU structure.
9108	* @param pVmxTransient The VMX-transient structure.
9109	* @param pEvent The event being injected.
9110	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
9111	* will be updated if necessary. This cannot not be NULL.
9112	* @param fStepping Whether we're single-stepping guest execution and should
9113	* return VINF_EM_DBG_STEPPED if the event is injected
9114	* directly (registers modified by us, not by hardware on
9115	* VM-entry).
9116	*/
9117	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
9118	uint32_t *pfIntrState)
9119	{
9120	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
9121	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
9122	Assert(pfIntrState);
9123
9124	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9125	uint32_t u32IntInfo = pEvent->u64IntInfo;
9126	uint32_t const u32ErrCode = pEvent->u32ErrCode;
9127	uint32_t const cbInstr = pEvent->cbInstr;
9128	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
9129	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
9130	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
9131
9132	#ifdef VBOX_STRICT
9133	/*
9134	* Validate the error-code-valid bit for hardware exceptions.
9135	* No error codes for exceptions in real-mode.
9136	*
9137	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9138	*/
9139	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9140	&& !CPUMIsGuestInRealModeEx(pCtx))
9141	{
9142	switch (uVector)
9143	{
9144	case X86_XCPT_PF:
9145	case X86_XCPT_DF:
9146	case X86_XCPT_TS:
9147	case X86_XCPT_NP:
9148	case X86_XCPT_SS:
9149	case X86_XCPT_GP:
9150	case X86_XCPT_AC:
9151	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
9152	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
9153	RT_FALL_THRU();
9154	default:
9155	break;
9156	}
9157	}
9158
9159	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
9160	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
9161	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9162	#endif
9163
9164	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
9165
9166	/*
9167	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
9168	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
9169	* interrupt handler in the (real-mode) guest.
9170	*
9171	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
9172	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
9173	*/
9174	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
9175	{
9176	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
9177	{
9178	/*
9179	* For CPUs with unrestricted guest execution enabled and with the guest
9180	* in real-mode, we must not set the deliver-error-code bit.
9181	*
9182	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
9183	*/
9184	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
9185	}
9186	else
9187	{
9188	PVM pVM = pVCpu->CTX_SUFF(pVM);
9189	Assert(PDMVmmDevHeapIsEnabled(pVM));
9190	Assert(pVM->hm.s.vmx.pRealModeTSS);
9191	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
9192
9193	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
9194	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9195	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
9196	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
9197	AssertRCReturn(rc2, rc2);
9198
9199	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
9200	size_t const cbIdtEntry = sizeof(X86IDTR16);
9201	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
9202	{
9203	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
9204	if (uVector == X86_XCPT_DF)
9205	return VINF_EM_RESET;
9206
9207	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
9208	No error codes for exceptions in real-mode. */
9209	if (uVector == X86_XCPT_GP)
9210	{
9211	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
9212	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9213	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9214	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9215	HMEVENT EventXcptDf;
9216	RT_ZERO(EventXcptDf);
9217	EventXcptDf.u64IntInfo = uXcptDfInfo;
9218	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
9219	}
9220
9221	/*
9222	* If we're injecting an event with no valid IDT entry, inject a #GP.
9223	* No error codes for exceptions in real-mode.
9224	*
9225	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9226	*/
9227	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
9228	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9229	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9230	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9231	HMEVENT EventXcptGp;
9232	RT_ZERO(EventXcptGp);
9233	EventXcptGp.u64IntInfo = uXcptGpInfo;
9234	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
9235	}
9236
9237	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
9238	uint16_t uGuestIp = pCtx->ip;
9239	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
9240	{
9241	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9242	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
9243	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9244	}
9245	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
9246	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9247
9248	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
9249	X86IDTR16 IdtEntry;
9250	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
9251	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
9252	AssertRCReturn(rc2, rc2);
9253
9254	/* Construct the stack frame for the interrupt/exception handler. */
9255	VBOXSTRICTRC rcStrict;
9256	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
9257	if (rcStrict == VINF_SUCCESS)
9258	{
9259	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
9260	if (rcStrict == VINF_SUCCESS)
9261	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
9262	}
9263
9264	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
9265	if (rcStrict == VINF_SUCCESS)
9266	{
9267	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
9268	pCtx->rip = IdtEntry.offSel;
9269	pCtx->cs.Sel = IdtEntry.uSel;
9270	pCtx->cs.ValidSel = IdtEntry.uSel;
9271	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
9272	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
9273	&& uVector == X86_XCPT_PF)
9274	pCtx->cr2 = GCPtrFault;
9275
9276	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
9277	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
9278	\| HM_CHANGED_GUEST_RSP);
9279
9280	/*
9281	* If we delivered a hardware exception (other than an NMI) and if there was
9282	* block-by-STI in effect, we should clear it.
9283	*/
9284	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9285	{
9286	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
9287	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
9288	Log4Func(("Clearing inhibition due to STI\n"));
9289	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
9290	}
9291
9292	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
9293	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
9294
9295	/*
9296	* The event has been truly dispatched to the guest. Mark it as no longer pending so
9297	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
9298	*/
9299	pVCpu->hm.s.Event.fPending = false;
9300
9301	/*
9302	* If we eventually support nested-guest execution without unrestricted guest execution,
9303	* we should set fInterceptEvents here.
9304	*/
9305	Assert(!pVmxTransient->fIsNestedGuest);
9306
9307	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
9308	if (fStepping)
9309	rcStrict = VINF_EM_DBG_STEPPED;
9310	}
9311	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9312	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9313	return rcStrict;
9314	}
9315	}
9316
9317	/*
9318	* Validate.
9319	*/
9320	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
9321	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
9322
9323	/*
9324	* Inject the event into the VMCS.
9325	*/
9326	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
9327	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
9328	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
9329	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
9330	AssertRCReturn(rc, rc);
9331
9332	/*
9333	* Update guest CR2 if this is a page-fault.
9334	*/
9335	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
9336	pCtx->cr2 = GCPtrFault;
9337
9338	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
9339	return VINF_SUCCESS;
9340	}
9341
9342
9343	/**
9344	* Evaluates the event to be delivered to the guest and sets it as the pending
9345	* event.
9346	*
9347	* @returns Strict VBox status code (i.e. informational status codes too).
9348	* @param pVCpu The cross context virtual CPU structure.
9349	* @param pVmxTransient The VMX-transient structure.
9350	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
9351	*/
9352	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
9353	{
9354	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9355	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9356	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
9357
9358	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
9359	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
9360	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9361	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9362	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9363
9364	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9365	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9366	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9367	Assert(!TRPMHasTrap(pVCpu));
9368	Assert(pfIntrState);
9369
9370	*pfIntrState = fIntrState;
9371
9372	/*
9373	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
9374	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
9375	*/
9376	/** @todo SMI. SMIs take priority over NMIs. */
9377	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
9378	{
9379	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
9380	if ( !pVCpu->hm.s.Event.fPending
9381	&& !fBlockNmi
9382	&& !fBlockSti
9383	&& !fBlockMovSS)
9384	{
9385	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9386	if ( fIsNestedGuest
9387	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
9388	return IEMExecVmxVmexitXcptNmi(pVCpu);
9389	#endif
9390	hmR0VmxSetPendingXcptNmi(pVCpu);
9391	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
9392	Log4Func(("Pending NMI\n"));
9393	}
9394	else if (!fIsNestedGuest)
9395	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
9396	}
9397	/*
9398	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
9399	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
9400	*/
9401	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
9402	&& !pVCpu->hm.s.fSingleInstruction)
9403	{
9404	Assert(!DBGFIsStepping(pVCpu));
9405	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9406	AssertRCReturn(rc, rc);
9407	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
9408	if ( !pVCpu->hm.s.Event.fPending
9409	&& !fBlockInt
9410	&& !fBlockSti
9411	&& !fBlockMovSS)
9412	{
9413	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9414	if ( fIsNestedGuest
9415	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
9416	{
9417	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
9418	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9419	return rcStrict;
9420	}
9421	#endif
9422	uint8_t u8Interrupt;
9423	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
9424	if (RT_SUCCESS(rc))
9425	{
9426	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9427	if ( fIsNestedGuest
9428	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
9429	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
9430	{
9431	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
9432	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9433	return rcStrict;
9434	}
9435	#endif
9436	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
9437	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
9438	}
9439	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
9440	{
9441	if ( !fIsNestedGuest
9442	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
9443	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
9444	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
9445
9446	/*
9447	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
9448	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
9449	* need to re-set this force-flag here.
9450	*/
9451	}
9452	else
9453	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
9454	}
9455	else if (!fIsNestedGuest)
9456	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
9457	}
9458
9459	return VINF_SUCCESS;
9460	}
9461
9462
9463	/**
9464	* Injects any pending events into the guest if the guest is in a state to
9465	* receive them.
9466	*
9467	* @returns Strict VBox status code (i.e. informational status codes too).
9468	* @param pVCpu The cross context virtual CPU structure.
9469	* @param pVmxTransient The VMX-transient structure.
9470	* @param fIntrState The VT-x guest-interruptibility state.
9471	* @param fStepping Whether we are single-stepping the guest using the
9472	* hypervisor debugger and should return
9473	* VINF_EM_DBG_STEPPED if the event was dispatched
9474	* directly.
9475	*/
9476	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9477	{
9478	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9479	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9480
9481	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9482	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9483
9484	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9485	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9486	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9487	Assert(!TRPMHasTrap(pVCpu));
9488
9489	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9490	if (pVCpu->hm.s.Event.fPending)
9491	{
9492	/*
9493	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9494	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9495	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9496	*
9497	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9498	*/
9499	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9500	#ifdef VBOX_STRICT
9501	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9502	{
9503	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9504	Assert(!fBlockInt);
9505	Assert(!fBlockSti);
9506	Assert(!fBlockMovSS);
9507	}
9508	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9509	{
9510	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9511	Assert(!fBlockSti);
9512	Assert(!fBlockMovSS);
9513	Assert(!fBlockNmi);
9514	}
9515	#endif
9516	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9517	uIntType));
9518
9519	/*
9520	* Inject the event and get any changes to the guest-interruptibility state.
9521	*
9522	* The guest-interruptibility state may need to be updated if we inject the event
9523	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9524	*/
9525	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9526	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9527
9528	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9529	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9530	else
9531	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9532	}
9533
9534	/*
9535	* Update the guest-interruptibility state.
9536	*
9537	* This is required for the real-on-v86 software interrupt injection case above, as well as
9538	* updates to the guest state from ring-3 or IEM/REM.
9539	*/
9540	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9541	AssertRCReturn(rc, rc);
9542
9543	/*
9544	* There's no need to clear the VM-entry interruption-information field here if we're not
9545	* injecting anything. VT-x clears the valid bit on every VM-exit.
9546	*
9547	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9548	*/
9549
9550	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9551	NOREF(fBlockMovSS); NOREF(fBlockSti);
9552	return rcStrict;
9553	}
9554
9555
9556	/**
9557	* Enters the VT-x session.
9558	*
9559	* @returns VBox status code.
9560	* @param pVCpu The cross context virtual CPU structure.
9561	*/
9562	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9563	{
9564	AssertPtr(pVCpu);
9565	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9566	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9567
9568	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9569	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9570	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9571
9572	#ifdef VBOX_STRICT
9573	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9574	RTCCUINTREG uHostCr4 = ASMGetCR4();
9575	if (!(uHostCr4 & X86_CR4_VMXE))
9576	{
9577	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9578	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9579	}
9580	#endif
9581
9582	/*
9583	* Load the appropriate VMCS as the current and active one.
9584	*/
9585	PVMXVMCSINFO pVmcsInfo;
9586	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9587	if (!fInNestedGuestMode)
9588	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9589	else
9590	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9591	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9592	if (RT_SUCCESS(rc))
9593	{
9594	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9595	pVCpu->hm.s.fLeaveDone = false;
9596	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9597
9598	/*
9599	* Do the EMT scheduled L1D flush here if needed.
9600	*/
9601	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9602	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9603	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9604	hmR0MdsClear();
9605	}
9606	return rc;
9607	}
9608
9609
9610	/**
9611	* The thread-context callback (only on platforms which support it).
9612	*
9613	* @param enmEvent The thread-context event.
9614	* @param pVCpu The cross context virtual CPU structure.
9615	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9616	* @thread EMT(pVCpu)
9617	*/
9618	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9619	{
9620	AssertPtr(pVCpu);
9621	RT_NOREF1(fGlobalInit);
9622
9623	switch (enmEvent)
9624	{
9625	case RTTHREADCTXEVENT_OUT:
9626	{
9627	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9628	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9629	VMCPU_ASSERT_EMT(pVCpu);
9630
9631	/* No longjmps (logger flushes, locks) in this fragile context. */
9632	VMMRZCallRing3Disable(pVCpu);
9633	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9634
9635	/* Restore host-state (FPU, debug etc.) */
9636	if (!pVCpu->hm.s.fLeaveDone)
9637	{
9638	/*
9639	* Do -not- import the guest-state here as we might already be in the middle of importing
9640	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9641	*/
9642	hmR0VmxLeave(pVCpu, false /* fImportState */);
9643	pVCpu->hm.s.fLeaveDone = true;
9644	}
9645
9646	/* Leave HM context, takes care of local init (term). */
9647	int rc = HMR0LeaveCpu(pVCpu);
9648	AssertRC(rc);
9649
9650	/* Restore longjmp state. */
9651	VMMRZCallRing3Enable(pVCpu);
9652	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9653	break;
9654	}
9655
9656	case RTTHREADCTXEVENT_IN:
9657	{
9658	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9659	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9660	VMCPU_ASSERT_EMT(pVCpu);
9661
9662	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9663	VMMRZCallRing3Disable(pVCpu);
9664	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9665
9666	/* Initialize the bare minimum state required for HM. This takes care of
9667	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9668	int rc = hmR0EnterCpu(pVCpu);
9669	AssertRC(rc);
9670	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9671	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9672
9673	/* Load the active VMCS as the current one. */
9674	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9675	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9676	AssertRC(rc);
9677	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9678	pVCpu->hm.s.fLeaveDone = false;
9679
9680	/* Do the EMT scheduled L1D flush if needed. */
9681	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9682	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9683
9684	/* Restore longjmp state. */
9685	VMMRZCallRing3Enable(pVCpu);
9686	break;
9687	}
9688
9689	default:
9690	break;
9691	}
9692	}
9693
9694
9695	/**
9696	* Exports the host state into the VMCS host-state area.
9697	* Sets up the VM-exit MSR-load area.
9698	*
9699	* The CPU state will be loaded from these fields on every successful VM-exit.
9700	*
9701	* @returns VBox status code.
9702	* @param pVCpu The cross context virtual CPU structure.
9703	*
9704	* @remarks No-long-jump zone!!!
9705	*/
9706	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9707	{
9708	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9709
9710	int rc = VINF_SUCCESS;
9711	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9712	{
9713	rc = hmR0VmxExportHostControlRegs();
9714	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9715
9716	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9717	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9718
9719	rc = hmR0VmxExportHostMsrs(pVCpu);
9720	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9721
9722	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9723	}
9724	return rc;
9725	}
9726
9727
9728	/**
9729	* Saves the host state in the VMCS host-state.
9730	*
9731	* @returns VBox status code.
9732	* @param pVCpu The cross context virtual CPU structure.
9733	*
9734	* @remarks No-long-jump zone!!!
9735	*/
9736	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9737	{
9738	AssertPtr(pVCpu);
9739	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9740
9741	/*
9742	* Export the host state here while entering HM context.
9743	* When thread-context hooks are used, we might get preempted and have to re-save the host
9744	* state but most of the time we won't be, so do it here before we disable interrupts.
9745	*/
9746	return hmR0VmxExportHostState(pVCpu);
9747	}
9748
9749
9750	/**
9751	* Exports the guest state into the VMCS guest-state area.
9752	*
9753	* The will typically be done before VM-entry when the guest-CPU state and the
9754	* VMCS state may potentially be out of sync.
9755	*
9756	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9757	* VM-entry controls.
9758	* Sets up the appropriate VMX non-root function to execute guest code based on
9759	* the guest CPU mode.
9760	*
9761	* @returns VBox strict status code.
9762	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9763	* without unrestricted guest execution and the VMMDev is not presently
9764	* mapped (e.g. EFI32).
9765	*
9766	* @param pVCpu The cross context virtual CPU structure.
9767	* @param pVmxTransient The VMX-transient structure.
9768	*
9769	* @remarks No-long-jump zone!!!
9770	*/
9771	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9772	{
9773	AssertPtr(pVCpu);
9774	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9775	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9776
9777	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9778
9779	/*
9780	* Determine real-on-v86 mode.
9781	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9782	*/
9783	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9784	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9785	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9786	pVmcsInfo->RealMode. fRealOnV86Active = false;
9787	else
9788	{
9789	Assert(!pVmxTransient->fIsNestedGuest);
9790	pVmcsInfo->RealMode.fRealOnV86Active = true;
9791	}
9792
9793	/*
9794	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9795	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9796	*/
9797	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9798	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9799	* be a need to evaluate this everytime since I'm pretty sure we intercept
9800	* all guest paging mode changes. */
9801	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9802	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9803
9804	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9805	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9806
9807	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9808	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9809
9810	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9811	if (rcStrict == VINF_SUCCESS)
9812	{ /* likely */ }
9813	else
9814	{
9815	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9816	return rcStrict;
9817	}
9818
9819	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9820	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9821
9822	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9823	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9824
9825	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9826	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9827
9828	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9829	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9830
9831	rc = hmR0VmxExportGuestRip(pVCpu);
9832	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9833	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9834	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9835
9836	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9837	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9838
9839	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9840	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9841	\| HM_CHANGED_GUEST_CR2
9842	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9843	\| HM_CHANGED_GUEST_X87
9844	\| HM_CHANGED_GUEST_SSE_AVX
9845	\| HM_CHANGED_GUEST_OTHER_XSAVE
9846	\| HM_CHANGED_GUEST_XCRx
9847	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9848	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9849	\| HM_CHANGED_GUEST_TSC_AUX
9850	\| HM_CHANGED_GUEST_OTHER_MSRS
9851	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9852
9853	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9854	return rc;
9855	}
9856
9857
9858	/**
9859	* Exports the state shared between the host and guest into the VMCS.
9860	*
9861	* @param pVCpu The cross context virtual CPU structure.
9862	* @param pVmxTransient The VMX-transient structure.
9863	*
9864	* @remarks No-long-jump zone!!!
9865	*/
9866	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9867	{
9868	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9869	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9870
9871	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9872	{
9873	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9874	AssertRC(rc);
9875	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9876
9877	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9878	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9879	{
9880	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9881	AssertRC(rc);
9882	}
9883	}
9884
9885	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9886	{
9887	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9888	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9889	}
9890
9891	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9892	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9893	}
9894
9895
9896	/**
9897	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9898	*
9899	* @returns Strict VBox status code (i.e. informational status codes too).
9900	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9901	* without unrestricted guest execution and the VMMDev is not presently
9902	* mapped (e.g. EFI32).
9903	*
9904	* @param pVCpu The cross context virtual CPU structure.
9905	* @param pVmxTransient The VMX-transient structure.
9906	*
9907	* @remarks No-long-jump zone!!!
9908	*/
9909	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9910	{
9911	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9912	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9913	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9914
9915	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9916	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9917	#endif
9918
9919	/*
9920	* For many exits it's only RIP that changes and hence try to export it first
9921	* without going through a lot of change flag checks.
9922	*/
9923	VBOXSTRICTRC rcStrict;
9924	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9925	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9926	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9927	{
9928	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9929	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9930	{ /* likely */}
9931	else
9932	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9934	}
9935	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9936	{
9937	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9938	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9939	{ /* likely */}
9940	else
9941	{
9942	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9943	VBOXSTRICTRC_VAL(rcStrict)));
9944	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9945	return rcStrict;
9946	}
9947	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9948	}
9949	else
9950	rcStrict = VINF_SUCCESS;
9951
9952	#ifdef VBOX_STRICT
9953	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9954	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9955	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9956	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9957	("fCtxChanged=%#RX64\n", fCtxChanged));
9958	#endif
9959	return rcStrict;
9960	}
9961
9962
9963	/**
9964	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9965	* and update error record fields accordingly.
9966	*
9967	* @returns VMX_IGS_* error codes.
9968	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9969	* wrong with the guest state.
9970	*
9971	* @param pVCpu The cross context virtual CPU structure.
9972	* @param pVmcsInfo The VMCS info. object.
9973	*
9974	* @remarks This function assumes our cache of the VMCS controls
9975	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9976	*/
9977	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9978	{
9979	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9980	#define HMVMX_CHECK_BREAK(expr, err) do { \
9981	if (!(expr)) { uError = (err); break; } \
9982	} while (0)
9983
9984	int rc;
9985	PVM pVM = pVCpu->CTX_SUFF(pVM);
9986	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9987	uint32_t uError = VMX_IGS_ERROR;
9988	uint32_t u32Val;
9989	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9990
9991	do
9992	{
9993	/*
9994	* CR0.
9995	*/
9996	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9997	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9998	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9999	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
10000	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10001	if (fUnrestrictedGuest)
10002	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10003
10004	uint32_t u32GuestCr0;
10005	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10006	AssertRCBreak(rc);
10007	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10008	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10009	if ( !fUnrestrictedGuest
10010	&& (u32GuestCr0 & X86_CR0_PG)
10011	&& !(u32GuestCr0 & X86_CR0_PE))
10012	{
10013	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10014	}
10015
10016	/*
10017	* CR4.
10018	*/
10019	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10020	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10021	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10022
10023	uint32_t u32GuestCr4;
10024	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10025	AssertRCBreak(rc);
10026	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10027	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10028
10029	/*
10030	* IA32_DEBUGCTL MSR.
10031	*/
10032	uint64_t u64Val;
10033	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10034	AssertRCBreak(rc);
10035	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10036	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10037	{
10038	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10039	}
10040	uint64_t u64DebugCtlMsr = u64Val;
10041
10042	#ifdef VBOX_STRICT
10043	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10044	AssertRCBreak(rc);
10045	Assert(u32Val == pVmcsInfo->u32EntryCtls);
10046	#endif
10047	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10048
10049	/*
10050	* RIP and RFLAGS.
10051	*/
10052	uint32_t u32Eflags;
10053	#if HC_ARCH_BITS == 64
10054	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10055	AssertRCBreak(rc);
10056	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10057	if ( !fLongModeGuest
10058	\|\| !pCtx->cs.Attr.n.u1Long)
10059	{
10060	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10061	}
10062	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10063	* must be identical if the "IA-32e mode guest" VM-entry
10064	* control is 1 and CS.L is 1. No check applies if the
10065	* CPU supports 64 linear-address bits. */
10066
10067	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10068	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10069	AssertRCBreak(rc);
10070	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10071	VMX_IGS_RFLAGS_RESERVED);
10072	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10073	u32Eflags = u64Val;
10074	#else
10075	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10076	AssertRCBreak(rc);
10077	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10078	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10079	#endif
10080
10081	if ( fLongModeGuest
10082	\|\| ( fUnrestrictedGuest
10083	&& !(u32GuestCr0 & X86_CR0_PE)))
10084	{
10085	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10086	}
10087
10088	uint32_t u32EntryInfo;
10089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10090	AssertRCBreak(rc);
10091	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
10092	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10093
10094	/*
10095	* 64-bit checks.
10096	*/
10097	#if HC_ARCH_BITS == 64
10098	if (fLongModeGuest)
10099	{
10100	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10101	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10102	}
10103
10104	if ( !fLongModeGuest
10105	&& (u32GuestCr4 & X86_CR4_PCIDE))
10106	{
10107	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10108	}
10109
10110	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10111	* 51:32 beyond the processor's physical-address width are 0. */
10112
10113	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10114	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10115	{
10116	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10117	}
10118
10119	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10120	AssertRCBreak(rc);
10121	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10122
10123	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10124	AssertRCBreak(rc);
10125	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10126	#endif
10127
10128	/*
10129	* PERF_GLOBAL MSR.
10130	*/
10131	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10132	{
10133	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10134	AssertRCBreak(rc);
10135	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10136	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10137	}
10138
10139	/*
10140	* PAT MSR.
10141	*/
10142	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10143	{
10144	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10145	AssertRCBreak(rc);
10146	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10147	for (unsigned i = 0; i < 8; i++)
10148	{
10149	uint8_t u8Val = (u64Val & 0xff);
10150	if ( u8Val != 0 /* UC */
10151	&& u8Val != 1 /* WC */
10152	&& u8Val != 4 /* WT */
10153	&& u8Val != 5 /* WP */
10154	&& u8Val != 6 /* WB */
10155	&& u8Val != 7 /* UC- */)
10156	{
10157	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10158	}
10159	u64Val >>= 8;
10160	}
10161	}
10162
10163	/*
10164	* EFER MSR.
10165	*/
10166	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10167	{
10168	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10169	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10170	AssertRCBreak(rc);
10171	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10172	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10173	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
10174	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10175	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10176	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10177	* iemVmxVmentryCheckGuestState(). */
10178	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10179	\|\| !(u32GuestCr0 & X86_CR0_PG)
10180	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10181	VMX_IGS_EFER_LMA_LME_MISMATCH);
10182	}
10183
10184	/*
10185	* Segment registers.
10186	*/
10187	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10188	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10189	if (!(u32Eflags & X86_EFL_VM))
10190	{
10191	/* CS */
10192	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10193	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10194	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10195	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10196	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10197	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10198	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10199	/* CS cannot be loaded with NULL in protected mode. */
10200	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10201	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10202	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10203	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10204	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10205	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10206	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10207	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10208	else
10209	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10210
10211	/* SS */
10212	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10213	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10214	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10215	if ( !(pCtx->cr0 & X86_CR0_PE)
10216	\|\| pCtx->cs.Attr.n.u4Type == 3)
10217	{
10218	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10219	}
10220	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10221	{
10222	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10223	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10224	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10225	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10226	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10227	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10228	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10229	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10230	}
10231
10232	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
10233	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10234	{
10235	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10236	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10237	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10238	\|\| pCtx->ds.Attr.n.u4Type > 11
10239	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10240	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10241	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10242	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10243	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10244	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10245	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10246	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10247	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10248	}
10249	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10250	{
10251	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10252	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10253	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10254	\|\| pCtx->es.Attr.n.u4Type > 11
10255	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10256	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10257	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10258	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10259	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10260	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10261	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10262	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10263	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10264	}
10265	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10266	{
10267	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10268	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10269	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10270	\|\| pCtx->fs.Attr.n.u4Type > 11
10271	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10272	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10273	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10274	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10275	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10276	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10277	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10278	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10279	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10280	}
10281	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10282	{
10283	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10284	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10285	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10286	\|\| pCtx->gs.Attr.n.u4Type > 11
10287	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10288	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10289	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10290	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10291	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10292	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10293	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10294	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10295	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10296	}
10297	/* 64-bit capable CPUs. */
10298	#if HC_ARCH_BITS == 64
10299	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10300	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10301	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10302	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10303	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10304	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10305	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10306	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10307	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10308	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10309	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10310	#endif
10311	}
10312	else
10313	{
10314	/* V86 mode checks. */
10315	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10316	if (pVmcsInfo->RealMode.fRealOnV86Active)
10317	{
10318	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10319	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10320	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10321	}
10322	else
10323	{
10324	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10325	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10326	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10327	}
10328
10329	/* CS */
10330	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10331	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10332	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10333	/* SS */
10334	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10335	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10336	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10337	/* DS */
10338	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10339	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10340	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10341	/* ES */
10342	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10343	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10344	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10345	/* FS */
10346	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10347	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10348	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10349	/* GS */
10350	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10351	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10352	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10353	/* 64-bit capable CPUs. */
10354	#if HC_ARCH_BITS == 64
10355	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10356	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10357	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10358	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10359	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10360	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10361	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10362	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10363	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10364	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10365	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10366	#endif
10367	}
10368
10369	/*
10370	* TR.
10371	*/
10372	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10373	/* 64-bit capable CPUs. */
10374	#if HC_ARCH_BITS == 64
10375	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10376	#endif
10377	if (fLongModeGuest)
10378	{
10379	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10380	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10381	}
10382	else
10383	{
10384	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10385	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10386	VMX_IGS_TR_ATTR_TYPE_INVALID);
10387	}
10388	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10389	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10390	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10391	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10392	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10393	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10394	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10395	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10396
10397	/*
10398	* GDTR and IDTR.
10399	*/
10400	#if HC_ARCH_BITS == 64
10401	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10402	AssertRCBreak(rc);
10403	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10404
10405	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10406	AssertRCBreak(rc);
10407	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10408	#endif
10409
10410	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10411	AssertRCBreak(rc);
10412	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10413
10414	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10415	AssertRCBreak(rc);
10416	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10417
10418	/*
10419	* Guest Non-Register State.
10420	*/
10421	/* Activity State. */
10422	uint32_t u32ActivityState;
10423	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10424	AssertRCBreak(rc);
10425	HMVMX_CHECK_BREAK( !u32ActivityState
10426	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10427	VMX_IGS_ACTIVITY_STATE_INVALID);
10428	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10429	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10430	uint32_t u32IntrState;
10431	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10432	AssertRCBreak(rc);
10433	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10434	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10435	{
10436	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10437	}
10438
10439	/** @todo Activity state and injecting interrupts. Left as a todo since we
10440	* currently don't use activity states but ACTIVE. */
10441
10442	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10443	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10444
10445	/* Guest interruptibility-state. */
10446	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10447	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10448	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10449	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10450	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10451	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10452	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10453	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
10454	{
10455	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10456	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10457	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10458	}
10459	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
10460	{
10461	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10462	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10463	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10464	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10465	}
10466	/** @todo Assumes the processor is not in SMM. */
10467	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10468	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10469	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10470	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10471	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10472	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10473	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
10474	{
10475	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10476	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10477	}
10478
10479	/* Pending debug exceptions. */
10480	#if HC_ARCH_BITS == 64
10481	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10482	AssertRCBreak(rc);
10483	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10484	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10485	u32Val = u64Val; /* For pending debug exceptions checks below. */
10486	#else
10487	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10488	AssertRCBreak(rc);
10489	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10490	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10491	#endif
10492
10493	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10494	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10495	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10496	{
10497	if ( (u32Eflags & X86_EFL_TF)
10498	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10499	{
10500	/* Bit 14 is PendingDebug.BS. */
10501	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10502	}
10503	if ( !(u32Eflags & X86_EFL_TF)
10504	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10505	{
10506	/* Bit 14 is PendingDebug.BS. */
10507	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10508	}
10509	}
10510
10511	/* VMCS link pointer. */
10512	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10513	AssertRCBreak(rc);
10514	if (u64Val != UINT64_C(0xffffffffffffffff))
10515	{
10516	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10517	/** @todo Bits beyond the processor's physical-address width MBZ. */
10518	/** @todo SMM checks. */
10519	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10520	Assert(pVmcsInfo->pvShadowVmcs);
10521	VMXVMCSREVID VmcsRevId;
10522	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10523	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10524	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10525	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10526	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10527	}
10528
10529	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10530	* not using nested paging? */
10531	if ( pVM->hm.s.fNestedPaging
10532	&& !fLongModeGuest
10533	&& CPUMIsGuestInPAEModeEx(pCtx))
10534	{
10535	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10536	AssertRCBreak(rc);
10537	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10538
10539	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10540	AssertRCBreak(rc);
10541	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10542
10543	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10544	AssertRCBreak(rc);
10545	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10546
10547	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10548	AssertRCBreak(rc);
10549	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10550	}
10551
10552	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10553	if (uError == VMX_IGS_ERROR)
10554	uError = VMX_IGS_REASON_NOT_FOUND;
10555	} while (0);
10556
10557	pVCpu->hm.s.u32HMError = uError;
10558	return uError;
10559
10560	#undef HMVMX_ERROR_BREAK
10561	#undef HMVMX_CHECK_BREAK
10562	}
10563
10564
10565	/**
10566	* Map the APIC-access page for virtualizing APIC accesses.
10567	*
10568	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10569	* this not done as part of exporting guest state, see @bugref{8721}.
10570	*
10571	* @returns VBox status code.
10572	* @param pVCpu The cross context virtual CPU structure.
10573	*/
10574	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10575	{
10576	PVM pVM = pVCpu->CTX_SUFF(pVM);
10577	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10578
10579	Assert(PDMHasApic(pVM));
10580	Assert(u64MsrApicBase);
10581
10582	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10583	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10584
10585	/* Unalias the existing mapping. */
10586	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10587	AssertRCReturn(rc, rc);
10588
10589	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10590	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10591	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10592	AssertRCReturn(rc, rc);
10593
10594	/* Update the per-VCPU cache of the APIC base MSR. */
10595	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10596	return VINF_SUCCESS;
10597	}
10598
10599
10600	/**
10601	* Worker function passed to RTMpOnSpecific() that is to be called on the target
10602	* CPU.
10603	*
10604	* @param idCpu The ID for the CPU the function is called on.
10605	* @param pvUser1 Null, not used.
10606	* @param pvUser2 Null, not used.
10607	*/
10608	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
10609	{
10610	RT_NOREF2(pvUser1, pvUser2);
10611	VMXDispatchHostNmi();
10612	}
10613
10614
10615	/**
10616	* Dispatching a host NMI on the host CPU that received the NMI.
10617	*
10618	* @returns VBox status code.
10619	* @param pVCpu The cross context virtual CPU structure.
10620	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
10621	* executing when receiving the host NMI.
10622	*/
10623	static int hmR0VmxExitHostNmi(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10624	{
10625	RTCPUID const idCpu = pVmcsInfo->idHostCpu;
10626
10627	/*
10628	* We don't want to delay dispatching the NMI any more than we have to. However,
10629	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
10630	* after executing guest or nested-guest code for the following reasons:
10631	*
10632	* - We would need to perform VMREADs with interrupts disabled and is orders of
10633	* magnitude worse with nested virtualization.
10634	*
10635	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
10636	* longer period of time just for handling an edge case like host NMIs.
10637	*
10638	* Let's cover the most likely scenario first. Check if we are on the target CPU
10639	* and dispatch the NMI right away. This should be much faster than calling into
10640	* RTMpOnSpecific() machinery.
10641	*/
10642	bool fDispatched = false;
10643	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10644	if (idCpu == RTMpCpuId())
10645	{
10646	VMXDispatchHostNmi();
10647	fDispatched = true;
10648	}
10649	ASMSetFlags(fEFlags);
10650	if (fDispatched)
10651	{
10652	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10653	return VINF_SUCCESS;
10654	}
10655
10656	/*
10657	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
10658	* there should be no race or recursion even if we are unlucky enough to be preempted
10659	* (to the target CPU) without dispatching the host NMI above.
10660	*/
10661	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
10662	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
10663	}
10664
10665
10666	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10667	/**
10668	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10669	* nested-guest using hardware-assisted VMX.
10670	*
10671	* @param pVCpu The cross context virtual CPU structure.
10672	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10673	* @param pVmcsInfoGst The guest VMCS info. object.
10674	*/
10675	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10676	{
10677	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10678	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10679	Assert(pu64MsrBitmap);
10680
10681	/*
10682	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10683	* MSR that is intercepted by the guest is also intercepted while executing the
10684	* nested-guest using hardware-assisted VMX.
10685	*
10686	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10687	* nested-guest VM-exit even if the outer guest is not intercepting some
10688	* MSRs. We cannot assume the caller has initialized the nested-guest
10689	* MSR bitmap in this case.
10690	*
10691	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10692	* each VM-entry, hence initializing it once per-VM while setting up the
10693	* nested-guest VMCS is not sufficient.
10694	*/
10695	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10696	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10697	{
10698	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10699	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10700	Assert(pu64MsrBitmapNstGst);
10701	Assert(pu64MsrBitmapGst);
10702
10703	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10704	for (uint32_t i = 0; i < cFrags; i++)
10705	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10706	}
10707	else
10708	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10709	}
10710
10711
10712	/**
10713	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10714	* hardware-assisted VMX execution of the nested-guest.
10715	*
10716	* For a guest, we don't modify these controls once we set up the VMCS and hence
10717	* this function is never called.
10718	*
10719	* For nested-guests since the guest hypervisor provides these controls on every
10720	* nested-guest VM-entry and could potentially change them everytime we need to
10721	* merge them before every nested-guest VM-entry.
10722	*
10723	* @returns VBox status code.
10724	* @param pVCpu The cross context virtual CPU structure.
10725	*/
10726	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10727	{
10728	PVM pVM = pVCpu->CTX_SUFF(pVM);
10729	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10730	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10731	Assert(pVmcsNstGst);
10732
10733	/*
10734	* Merge the controls with the requirements of the guest VMCS.
10735	*
10736	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10737	* VMCS with the features supported by the physical CPU as it's already done by the
10738	* VMLAUNCH/VMRESUME instruction emulation.
10739	*
10740	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10741	* derived from the VMX features supported by the physical CPU.
10742	*/
10743
10744	/* Pin-based VM-execution controls. */
10745	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10746
10747	/* Processor-based VM-execution controls. */
10748	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10749	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10750	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10751	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10752	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10753
10754	/* Secondary processor-based VM-execution controls. */
10755	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10756	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10757	\| VMX_PROC_CTLS2_INVPCID
10758	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10759	\| VMX_PROC_CTLS2_RDTSCP
10760	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10761	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10762	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10763	\| VMX_PROC_CTLS2_VMFUNC));
10764
10765	/*
10766	* VM-entry controls:
10767	* These controls contains state that depends on the nested-guest state (primarily
10768	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10769	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10770	* properly continue executing the nested-guest if the EFER MSR changes but does not
10771	* cause a nested-guest VM-exits.
10772	*
10773	* VM-exit controls:
10774	* These controls specify the host state on return. We cannot use the controls from
10775	* the guest hypervisor state as is as it would contain the guest state rather than
10776	* the host state. Since the host state is subject to change (e.g. preemption, trips
10777	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10778	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10779	*
10780	* VM-entry MSR-load:
10781	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10782	* context by the VMLAUNCH/VMRESUME instruction emulation.
10783	*
10784	* VM-exit MSR-store:
10785	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10786	* back into the VM-exit MSR-store area.
10787	*
10788	* VM-exit MSR-load areas:
10789	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10790	* can entirely ignore what the guest hypervisor wants to load here.
10791	*/
10792
10793	/*
10794	* Exception bitmap.
10795	*
10796	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10797	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10798	* code more flexible if intercepting exceptions become more dynamic in the future we do
10799	* it as part of exporting the nested-guest state.
10800	*/
10801	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10802
10803	/*
10804	* CR0/CR4 guest/host mask.
10805	*
10806	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10807	* cause VM-exits, so we need to merge them here.
10808	*/
10809	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10810	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10811
10812	/*
10813	* Page-fault error-code mask and match.
10814	*
10815	* Although we require unrestricted guest execution (and thereby nested-paging) for
10816	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10817	* normally intercept #PFs, it might intercept them for debugging purposes.
10818	*
10819	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10820	* If the outer guest is intercepting #PFs we must intercept all #PFs.
10821	*/
10822	uint32_t u32XcptPFMask;
10823	uint32_t u32XcptPFMatch;
10824	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10825	{
10826	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10827	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10828	}
10829	else
10830	{
10831	u32XcptPFMask = 0;
10832	u32XcptPFMatch = 0;
10833	}
10834
10835	/*
10836	* Pause-Loop exiting.
10837	*/
10838	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10839	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10840
10841	/*
10842	* I/O Bitmap.
10843	*
10844	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we always
10845	* intercept all I/O port accesses.
10846	*/
10847	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10848
10849	/*
10850	* VMCS shadowing.
10851	*
10852	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10853	* enabled while executing the nested-guest.
10854	*/
10855	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10856
10857	/*
10858	* APIC-access page.
10859	*
10860	* The APIC-access page address has already been initialized while setting up the
10861	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it should
10862	* not be of any consequence to the host or to the guest for that matter, but we only
10863	* accept valid addresses verified by the VMLAUNCH/VMRESUME instruction emulation to keep
10864	* it simple.
10865	*/
10866
10867	/*
10868	* Virtual-APIC page and TPR threshold.
10869	*
10870	* The virtual-APIC page has already been allocated (by CPUM during VM startup) and cached
10871	* from guest memory as part of VMLAUNCH/VMRESUME instruction emulation. The host physical
10872	* address has also been updated in the nested-guest VMCS.
10873	*/
10874	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10875	RTHCPHYS HCPhysVirtApic;
10876	uint32_t u32TprThreshold;
10877	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10878	{
10879	HCPhysVirtApic = pVmcsInfoNstGst->HCPhysVirtApic;
10880	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10881	}
10882	else
10883	{
10884	HCPhysVirtApic = 0;
10885	u32TprThreshold = 0;
10886
10887	/*
10888	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10889	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10890	* be taken care of by EPT/shadow paging.
10891	*/
10892	if (pVM->hm.s.fAllow64BitGuests)
10893	{
10894	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10895	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10896	}
10897	}
10898
10899	/*
10900	* Validate basic assumptions.
10901	*/
10902	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10903	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10904	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10905
10906	/*
10907	* Commit it to the nested-guest VMCS.
10908	*/
10909	int rc = VINF_SUCCESS;
10910	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10911	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10912	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10913	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10914	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10915	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10916	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10917	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10918	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10919	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10920	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10921	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10922	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10923	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10924	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10925	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10926	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10927	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10928	{
10929	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10930	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10931	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10932	}
10933	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10934	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVmcsInfoNstGst->HCPhysVirtApic);
10935	AssertRCReturn(rc, rc);
10936
10937	/*
10938	* Update the nested-guest VMCS cache.
10939	*/
10940	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10941	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10942	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10943	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10944	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10945	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10946	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10947	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10948	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10949
10950	/*
10951	* MSR bitmap.
10952	*
10953	* The MSR bitmap address has already been initialized while setting up the nested-guest
10954	* VMCS, here we need to merge the MSR bitmaps.
10955	*/
10956	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10957	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10958
10959	return VINF_SUCCESS;
10960	}
10961	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10962
10963
10964	/**
10965	* Does the preparations before executing guest code in VT-x.
10966	*
10967	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10968	* recompiler/IEM. We must be cautious what we do here regarding committing
10969	* guest-state information into the VMCS assuming we assuredly execute the
10970	* guest in VT-x mode.
10971	*
10972	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10973	* the common-state (TRPM/forceflags), we must undo those changes so that the
10974	* recompiler/IEM can (and should) use them when it resumes guest execution.
10975	* Otherwise such operations must be done when we can no longer exit to ring-3.
10976	*
10977	* @returns Strict VBox status code (i.e. informational status codes too).
10978	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10979	* have been disabled.
10980	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10981	* pending events).
10982	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10983	* double-fault into the guest.
10984	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10985	* dispatched directly.
10986	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10987	*
10988	* @param pVCpu The cross context virtual CPU structure.
10989	* @param pVmxTransient The VMX-transient structure.
10990	* @param fStepping Whether we are single-stepping the guest in the
10991	* hypervisor debugger. Makes us ignore some of the reasons
10992	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10993	* if event dispatching took place.
10994	*/
10995	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10996	{
10997	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10998
10999	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
11000	if (pVmxTransient->fIsNestedGuest)
11001	{
11002	RT_NOREF2(pVCpu, fStepping);
11003	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
11004	return VINF_EM_RESCHEDULE_REM;
11005	}
11006	#endif
11007
11008	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
11009	PGMRZDynMapFlushAutoSet(pVCpu);
11010	#endif
11011
11012	/*
11013	* Check and process force flag actions, some of which might require us to go back to ring-3.
11014	*/
11015	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
11016	if (rcStrict == VINF_SUCCESS)
11017	{ /* FFs don't get set all the time. */ }
11018	else
11019	return rcStrict;
11020
11021	/*
11022	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
11023	*/
11024	/** @todo Doing this from ring-3 after VM setup phase causes a
11025	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
11026	* idea why atm. */
11027	PVM pVM = pVCpu->CTX_SUFF(pVM);
11028	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
11029	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
11030	&& PDMHasApic(pVM))
11031	{
11032	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
11033	AssertRCReturn(rc, rc);
11034	}
11035
11036	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11037	/*
11038	* Merge guest VMCS controls with the nested-guest VMCS controls.
11039	*
11040	* Even if we have not executed the guest prior to this (e.g. when resuming from a
11041	* saved state), we should be okay with merging controls as we initialize the
11042	* guest VMCS controls as part of VM setup phase.
11043	*/
11044	if ( pVmxTransient->fIsNestedGuest
11045	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
11046	{
11047	int rc = hmR0VmxMergeVmcsNested(pVCpu);
11048	AssertRCReturn(rc, rc);
11049	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
11050	}
11051	#endif
11052
11053	/*
11054	* Evaluate events to be injected into the guest.
11055	*
11056	* Events in TRPM can be injected without inspecting the guest state.
11057	* If any new events (interrupts/NMI) are pending currently, we try to set up the
11058	* guest to cause a VM-exit the next time they are ready to receive the event.
11059	*
11060	* With nested-guests, evaluating pending events may cause VM-exits.
11061	*/
11062	if (TRPMHasTrap(pVCpu))
11063	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
11064
11065	uint32_t fIntrState;
11066	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
11067
11068	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11069	/*
11070	* While evaluating pending events if something failed (unlikely) or if we were
11071	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
11072	*/
11073	if (rcStrict != VINF_SUCCESS)
11074	return rcStrict;
11075	if ( pVmxTransient->fIsNestedGuest
11076	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11077	{
11078	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11079	return VINF_VMX_VMEXIT;
11080	}
11081	#else
11082	Assert(rcStrict == VINF_SUCCESS);
11083	#endif
11084
11085	/*
11086	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
11087	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
11088	* also result in triple-faulting the VM.
11089	*
11090	* With nested-guests, the above does not apply since unrestricted guest execution is a
11091	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
11092	*/
11093	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
11094	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11095	{ /* likely */ }
11096	else
11097	{
11098	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
11099	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11100	return rcStrict;
11101	}
11102
11103	/*
11104	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
11105	* import CR3 themselves. We will need to update them here, as even as late as the above
11106	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
11107	* the below force flags to be set.
11108	*/
11109	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
11110	{
11111	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
11112	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
11113	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
11114	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
11115	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11116	}
11117	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
11118	{
11119	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
11120	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11121	}
11122
11123	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11124	/* Paranoia. */
11125	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11126	#endif
11127
11128	/*
11129	* No longjmps to ring-3 from this point on!!!
11130	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
11131	* This also disables flushing of the R0-logger instance (if any).
11132	*/
11133	VMMRZCallRing3Disable(pVCpu);
11134
11135	/*
11136	* Export the guest state bits.
11137	*
11138	* We cannot perform longjmps while loading the guest state because we do not preserve the
11139	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
11140	* CPU migration.
11141	*
11142	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
11143	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
11144	*/
11145	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
11146	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11147	{ /* likely */ }
11148	else
11149	{
11150	VMMRZCallRing3Enable(pVCpu);
11151	return rcStrict;
11152	}
11153
11154	/*
11155	* We disable interrupts so that we don't miss any interrupts that would flag preemption
11156	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
11157	* preemption disabled for a while. Since this is purely to aid the
11158	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
11159	* disable interrupt on NT.
11160	*
11161	* We need to check for force-flags that could've possible been altered since we last
11162	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
11163	* see @bugref{6398}).
11164	*
11165	* We also check a couple of other force-flags as a last opportunity to get the EMT back
11166	* to ring-3 before executing guest code.
11167	*/
11168	pVmxTransient->fEFlags = ASMIntDisableFlags();
11169
11170	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
11171	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
11172	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
11173	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
11174	{
11175	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
11176	{
11177	pVCpu->hm.s.Event.fPending = false;
11178
11179	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11180	/*
11181	* If we are executing a nested-guest make sure that we should intercept subsequent
11182	* events. The one we are injecting might be part of VM-entry.
11183	*/
11184	if (pVmxTransient->fIsNestedGuest)
11185	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
11186	#endif
11187
11188	/*
11189	* We've injected any pending events. This is really the point of no return (to ring-3).
11190	*
11191	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
11192	* returns from this function, so don't enable them here.
11193	*/
11194	return VINF_SUCCESS;
11195	}
11196
11197	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
11198	rcStrict = VINF_EM_RAW_INTERRUPT;
11199	}
11200	else
11201	{
11202	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
11203	rcStrict = VINF_EM_RAW_TO_R3;
11204	}
11205
11206	ASMSetFlags(pVmxTransient->fEFlags);
11207	VMMRZCallRing3Enable(pVCpu);
11208
11209	return rcStrict;
11210	}
11211
11212
11213	/**
11214	* Final preparations before executing guest code using hardware-assisted VMX.
11215	*
11216	* We can no longer get preempted to a different host CPU and there are no returns
11217	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
11218	* failures), this function is not intended to fail sans unrecoverable hardware
11219	* errors.
11220	*
11221	* @param pVCpu The cross context virtual CPU structure.
11222	* @param pVmxTransient The VMX-transient structure.
11223	*
11224	* @remarks Called with preemption disabled.
11225	* @remarks No-long-jump zone!!!
11226	*/
11227	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11228	{
11229	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11230	Assert(VMMR0IsLogFlushDisabled(pVCpu));
11231	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
11232	Assert(!pVCpu->hm.s.Event.fPending);
11233
11234	/*
11235	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
11236	*/
11237	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11238	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
11239
11240	PVM pVM = pVCpu->CTX_SUFF(pVM);
11241	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11242	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
11243	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
11244
11245	if (!CPUMIsGuestFPUStateActive(pVCpu))
11246	{
11247	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11248	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
11249	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
11250	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11251	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
11252	}
11253
11254	/*
11255	* Re-export the host state bits as we may've been preempted (only happens when
11256	* thread-context hooks are used or when the VM start function changes) or if
11257	* the host CR0 is modified while loading the guest FPU state above.
11258	*
11259	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
11260	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
11261	* see @bugref{8432}.
11262	*
11263	* This may also happen when switching to/from a nested-guest VMCS without leaving
11264	* ring-0.
11265	*/
11266	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
11267	{
11268	hmR0VmxExportHostState(pVCpu);
11269	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
11270	}
11271	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
11272
11273	/*
11274	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
11275	*/
11276	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
11277	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
11278	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
11279
11280	/*
11281	* Store status of the shared guest/host debug state at the time of VM-entry.
11282	*/
11283	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
11284	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
11285	{
11286	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
11287	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
11288	}
11289	else
11290	#endif
11291	{
11292	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
11293	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
11294	}
11295
11296	/*
11297	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
11298	* more than one conditional check. The post-run side of our code shall determine
11299	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
11300	*/
11301	if (pVmcsInfo->pbVirtApic)
11302	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
11303
11304	/*
11305	* Update the host MSRs values in the VM-exit MSR-load area.
11306	*/
11307	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
11308	{
11309	if (pVmcsInfo->cExitMsrLoad > 0)
11310	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
11311	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
11312	}
11313
11314	/*
11315	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
11316	* VMX-preemption timer based on the next virtual sync clock deadline.
11317	*/
11318	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
11319	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
11320	{
11321	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
11322	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
11323	}
11324
11325	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
11326	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
11327	if (!fIsRdtscIntercepted)
11328	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
11329	else
11330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
11331
11332	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
11333	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
11334	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
11335	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
11336	pVmcsInfo->idHostCpu = idCurrentCpu; /* Update the CPU for which we updated host-state in this VMCS. */
11337
11338	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
11339
11340	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
11341	as we're about to start executing the guest . */
11342
11343	/*
11344	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
11345	*
11346	* This is done this late as updating the TSC offsetting/preemption timer above
11347	* figures out if we can skip intercepting RDTSCP by calculating the number of
11348	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
11349	*/
11350	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
11351	&& !fIsRdtscIntercepted)
11352	{
11353	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
11354
11355	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
11356	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
11357	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
11358	true /* fSetReadWrite /, true / fUpdateHostMsr */);
11359	AssertRC(rc);
11360	Assert(!pVmxTransient->fRemoveTscAuxMsr);
11361	pVmxTransient->fRemoveTscAuxMsr = true;
11362	}
11363
11364	#ifdef VBOX_STRICT
11365	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
11366	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11367	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
11368	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
11369	#endif
11370
11371	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
11372	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
11373	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
11374	* see @bugref{9180#c54}. */
11375	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
11376	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
11377	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
11378	#endif
11379	}
11380
11381
11382	/**
11383	* First C routine invoked after running guest code using hardware-assisted VMX.
11384	*
11385	* @param pVCpu The cross context virtual CPU structure.
11386	* @param pVmxTransient The VMX-transient structure.
11387	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
11388	*
11389	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
11390	*
11391	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
11392	* unconditionally when it is safe to do so.
11393	*/
11394	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
11395	{
11396	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
11397
11398	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
11399	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
11400	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
11401	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
11402	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
11403	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
11404
11405	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11406	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11407	{
11408	uint64_t uGstTsc;
11409	if (!pVmxTransient->fIsNestedGuest)
11410	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11411	else
11412	{
11413	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11414	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
11415	}
11416	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
11417	}
11418
11419	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11420	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
11421	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11422
11423	#if HC_ARCH_BITS == 64
11424	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11425	#endif
11426	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
11427	/* The 64-on-32 switcher maintains VMCS-launch state on its own
11428	and we need to leave it alone here. */
11429	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
11430	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11431	#else
11432	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11433	#endif
11434	#ifdef VBOX_STRICT
11435	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11436	#endif
11437	Assert(!ASMIntAreEnabled());
11438	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11439	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11440
11441	/*
11442	* Save the basic VM-exit reason and check if the VM-entry failed.
11443	* See Intel spec. 24.9.1 "Basic VM-exit Information".
11444	*/
11445	uint32_t uExitReason;
11446	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11447	AssertRC(rc);
11448	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11449	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11450
11451	/*
11452	* Log the VM-exit before logging anything else as otherwise it might be a
11453	* tad confusing what happens before and after the world-switch.
11454	*/
11455	HMVMX_LOG_EXIT(pVCpu, uExitReason);
11456
11457	/*
11458	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11459	* bitmap permissions, if it was added before VM-entry.
11460	*/
11461	if (pVmxTransient->fRemoveTscAuxMsr)
11462	{
11463	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11464	pVmxTransient->fRemoveTscAuxMsr = false;
11465	}
11466
11467	/*
11468	* Check if VMLAUNCH/VMRESUME succeeded.
11469	* If this failed, we cause a guru meditation and cease further execution.
11470	*
11471	* However, if we are executing a nested-guest we might fail if we use the
11472	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11473	*/
11474	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11475	{
11476	/*
11477	* Update the VM-exit history array here even if the VM-entry failed due to:
11478	* - Invalid guest state.
11479	* - MSR loading.
11480	* - Machine-check event.
11481	*
11482	* In any of the above cases we will still have a "valid" VM-exit reason
11483	* despite @a fVMEntryFailed being false.
11484	*
11485	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11486	*
11487	* Note! We don't have CS or RIP at this point. Will probably address that later
11488	* by amending the history entry added here.
11489	*/
11490	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11491	UINT64_MAX, uHostTsc);
11492
11493	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11494	{
11495	VMMRZCallRing3Enable(pVCpu);
11496
11497	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11498	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11499
11500	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11501	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11502	AssertRC(rc);
11503	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11504	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11505	AssertRC(rc);
11506	#else
11507	/*
11508	* Import the guest-interruptibility state always as we need it while evaluating
11509	* injecting events on re-entry.
11510	*
11511	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11512	* checking for real-mode while exporting the state because all bits that cause
11513	* mode changes wrt CR0 are intercepted.
11514	*/
11515	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11516	AssertRC(rc);
11517	#endif
11518
11519	/*
11520	* Sync the TPR shadow with our APIC state.
11521	*
11522	* With nested-guests, mark the virtual-APIC page as dirty so it can be synced
11523	* when performing the nested-guest VM-exit.
11524	*/
11525	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
11526	{
11527	if (!pVmxTransient->fIsNestedGuest)
11528	{
11529	Assert(pVmcsInfo->pbVirtApic);
11530	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11531	{
11532	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11533	AssertRC(rc);
11534	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11535	}
11536	}
11537	else
11538	pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtApicPageDirty = true;
11539	}
11540
11541	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11542	return;
11543	}
11544	}
11545	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11546	else if (pVmxTransient->fIsNestedGuest)
11547	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11548	#endif
11549	else
11550	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11551
11552	VMMRZCallRing3Enable(pVCpu);
11553	}
11554
11555
11556	/**
11557	* Runs the guest code using hardware-assisted VMX the normal way.
11558	*
11559	* @returns VBox status code.
11560	* @param pVCpu The cross context virtual CPU structure.
11561	* @param pcLoops Pointer to the number of executed loops.
11562	*/
11563	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11564	{
11565	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11566	Assert(pcLoops);
11567	Assert(*pcLoops <= cMaxResumeLoops);
11568	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11569
11570	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11571	/*
11572	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11573	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11574	* guest VMCS while entering the VMX ring-0 session.
11575	*/
11576	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11577	{
11578	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11579	if (RT_SUCCESS(rc))
11580	{ /* likely */ }
11581	else
11582	{
11583	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11584	return rc;
11585	}
11586	}
11587	#endif
11588
11589	VMXTRANSIENT VmxTransient;
11590	RT_ZERO(VmxTransient);
11591	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11592
11593	/* Paranoia. */
11594	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11595
11596	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11597	for (;;)
11598	{
11599	Assert(!HMR0SuspendPending());
11600	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11601	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11602
11603	/*
11604	* Preparatory work for running nested-guest code, this may force us to
11605	* return to ring-3.
11606	*
11607	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11608	*/
11609	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11610	if (rcStrict != VINF_SUCCESS)
11611	break;
11612
11613	/* Interrupts are disabled at this point! */
11614	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11615	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11616	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11617	/* Interrupts are re-enabled at this point! */
11618
11619	/*
11620	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11621	*/
11622	if (RT_SUCCESS(rcRun))
11623	{ /* very likely */ }
11624	else
11625	{
11626	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11627	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11628	return rcRun;
11629	}
11630
11631	/*
11632	* Profile the VM-exit.
11633	*/
11634	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11635	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11636	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11637	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11638	HMVMX_START_EXIT_DISPATCH_PROF();
11639
11640	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11641
11642	/*
11643	* Handle the VM-exit.
11644	*/
11645	#ifdef HMVMX_USE_FUNCTION_TABLE
11646	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11647	#else
11648	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11649	#endif
11650	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11651	if (rcStrict == VINF_SUCCESS)
11652	{
11653	if (++(*pcLoops) <= cMaxResumeLoops)
11654	continue;
11655	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11656	rcStrict = VINF_EM_RAW_INTERRUPT;
11657	}
11658	break;
11659	}
11660
11661	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11662	return rcStrict;
11663	}
11664
11665
11666	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11667	/**
11668	* Runs the nested-guest code using hardware-assisted VMX.
11669	*
11670	* @returns VBox status code.
11671	* @param pVCpu The cross context virtual CPU structure.
11672	* @param pcLoops Pointer to the number of executed loops.
11673	*
11674	* @sa hmR0VmxRunGuestCodeNormal.
11675	*/
11676	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11677	{
11678	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11679	Assert(pcLoops);
11680	Assert(*pcLoops <= cMaxResumeLoops);
11681	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11682
11683	/*
11684	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11685	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11686	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11687	*/
11688	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11689	{
11690	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11691	if (RT_SUCCESS(rc))
11692	{ /* likely */ }
11693	else
11694	{
11695	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11696	return rc;
11697	}
11698	}
11699
11700	VMXTRANSIENT VmxTransient;
11701	RT_ZERO(VmxTransient);
11702	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11703	VmxTransient.fIsNestedGuest = true;
11704
11705	/* Paranoia. */
11706	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11707
11708	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11709	for (;;)
11710	{
11711	Assert(!HMR0SuspendPending());
11712	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11713	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11714
11715	/*
11716	* Preparatory work for running guest code, this may force us to
11717	* return to ring-3.
11718	*
11719	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11720	*/
11721	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11722	if (rcStrict != VINF_SUCCESS)
11723	break;
11724
11725	/* Interrupts are disabled at this point! */
11726	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11727	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11728	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11729	/* Interrupts are re-enabled at this point! */
11730
11731	/*
11732	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11733	*/
11734	if (RT_SUCCESS(rcRun))
11735	{ /* very likely */ }
11736	else
11737	{
11738	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11739	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11740	return rcRun;
11741	}
11742
11743	/*
11744	* Profile the VM-exit.
11745	*/
11746	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11747	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11748	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11749	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11750	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11751	HMVMX_START_EXIT_DISPATCH_PROF();
11752
11753	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11754
11755	/*
11756	* Handle the VM-exit.
11757	*/
11758	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11759	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11760	if (rcStrict == VINF_SUCCESS)
11761	{
11762	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11763	{
11764	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11765	rcStrict = VINF_VMX_VMEXIT;
11766	}
11767	else
11768	{
11769	if (++(*pcLoops) <= cMaxResumeLoops)
11770	continue;
11771	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11772	rcStrict = VINF_EM_RAW_INTERRUPT;
11773	}
11774	}
11775	else
11776	Assert(rcStrict != VINF_VMX_VMEXIT);
11777	break;
11778	}
11779
11780	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11781	return rcStrict;
11782	}
11783	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11784
11785
11786	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11787	* probes.
11788	*
11789	* The following few functions and associated structure contains the bloat
11790	* necessary for providing detailed debug events and dtrace probes as well as
11791	* reliable host side single stepping. This works on the principle of
11792	* "subclassing" the normal execution loop and workers. We replace the loop
11793	* method completely and override selected helpers to add necessary adjustments
11794	* to their core operation.
11795	*
11796	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11797	* any performance for debug and analysis features.
11798	*
11799	* @{
11800	*/
11801
11802	/**
11803	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11804	* the debug run loop.
11805	*/
11806	typedef struct VMXRUNDBGSTATE
11807	{
11808	/** The RIP we started executing at. This is for detecting that we stepped. */
11809	uint64_t uRipStart;
11810	/** The CS we started executing with. */
11811	uint16_t uCsStart;
11812
11813	/** Whether we've actually modified the 1st execution control field. */
11814	bool fModifiedProcCtls : 1;
11815	/** Whether we've actually modified the 2nd execution control field. */
11816	bool fModifiedProcCtls2 : 1;
11817	/** Whether we've actually modified the exception bitmap. */
11818	bool fModifiedXcptBitmap : 1;
11819
11820	/** We desire the modified the CR0 mask to be cleared. */
11821	bool fClearCr0Mask : 1;
11822	/** We desire the modified the CR4 mask to be cleared. */
11823	bool fClearCr4Mask : 1;
11824	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11825	uint32_t fCpe1Extra;
11826	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11827	uint32_t fCpe1Unwanted;
11828	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11829	uint32_t fCpe2Extra;
11830	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11831	uint32_t bmXcptExtra;
11832	/** The sequence number of the Dtrace provider settings the state was
11833	* configured against. */
11834	uint32_t uDtraceSettingsSeqNo;
11835	/** VM-exits to check (one bit per VM-exit). */
11836	uint32_t bmExitsToCheck[3];
11837
11838	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11839	uint32_t fProcCtlsInitial;
11840	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11841	uint32_t fProcCtls2Initial;
11842	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11843	uint32_t bmXcptInitial;
11844	} VMXRUNDBGSTATE;
11845	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11846	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11847
11848
11849	/**
11850	* Initializes the VMXRUNDBGSTATE structure.
11851	*
11852	* @param pVCpu The cross context virtual CPU structure of the
11853	* calling EMT.
11854	* @param pVmxTransient The VMX-transient structure.
11855	* @param pDbgState The debug state to initialize.
11856	*/
11857	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11858	{
11859	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11860	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11861
11862	pDbgState->fModifiedProcCtls = false;
11863	pDbgState->fModifiedProcCtls2 = false;
11864	pDbgState->fModifiedXcptBitmap = false;
11865	pDbgState->fClearCr0Mask = false;
11866	pDbgState->fClearCr4Mask = false;
11867	pDbgState->fCpe1Extra = 0;
11868	pDbgState->fCpe1Unwanted = 0;
11869	pDbgState->fCpe2Extra = 0;
11870	pDbgState->bmXcptExtra = 0;
11871	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11872	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11873	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11874	}
11875
11876
11877	/**
11878	* Updates the VMSC fields with changes requested by @a pDbgState.
11879	*
11880	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11881	* immediately before executing guest code, i.e. when interrupts are disabled.
11882	* We don't check status codes here as we cannot easily assert or return in the
11883	* latter case.
11884	*
11885	* @param pVCpu The cross context virtual CPU structure.
11886	* @param pVmxTransient The VMX-transient structure.
11887	* @param pDbgState The debug state.
11888	*/
11889	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11890	{
11891	/*
11892	* Ensure desired flags in VMCS control fields are set.
11893	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11894	*
11895	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11896	* there should be no stale data in pCtx at this point.
11897	*/
11898	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11899	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11900	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11901	{
11902	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11903	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11904	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11905	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11906	pDbgState->fModifiedProcCtls = true;
11907	}
11908
11909	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11910	{
11911	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11912	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11913	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11914	pDbgState->fModifiedProcCtls2 = true;
11915	}
11916
11917	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11918	{
11919	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11920	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11921	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11922	pDbgState->fModifiedXcptBitmap = true;
11923	}
11924
11925	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11926	{
11927	pVmcsInfo->u64Cr0Mask = 0;
11928	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11929	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11930	}
11931
11932	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11933	{
11934	pVmcsInfo->u64Cr4Mask = 0;
11935	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11936	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11937	}
11938
11939	NOREF(pVCpu);
11940	}
11941
11942
11943	/**
11944	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11945	* re-entry next time around.
11946	*
11947	* @returns Strict VBox status code (i.e. informational status codes too).
11948	* @param pVCpu The cross context virtual CPU structure.
11949	* @param pVmxTransient The VMX-transient structure.
11950	* @param pDbgState The debug state.
11951	* @param rcStrict The return code from executing the guest using single
11952	* stepping.
11953	*/
11954	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11955	VBOXSTRICTRC rcStrict)
11956	{
11957	/*
11958	* Restore VM-exit control settings as we may not reenter this function the
11959	* next time around.
11960	*/
11961	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11962
11963	/* We reload the initial value, trigger what we can of recalculations the
11964	next time around. From the looks of things, that's all that's required atm. */
11965	if (pDbgState->fModifiedProcCtls)
11966	{
11967	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11968	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11969	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11970	AssertRCReturn(rc2, rc2);
11971	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11972	}
11973
11974	/* We're currently the only ones messing with this one, so just restore the
11975	cached value and reload the field. */
11976	if ( pDbgState->fModifiedProcCtls2
11977	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11978	{
11979	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11980	AssertRCReturn(rc2, rc2);
11981	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11982	}
11983
11984	/* If we've modified the exception bitmap, we restore it and trigger
11985	reloading and partial recalculation the next time around. */
11986	if (pDbgState->fModifiedXcptBitmap)
11987	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11988
11989	return rcStrict;
11990	}
11991
11992
11993	/**
11994	* Configures VM-exit controls for current DBGF and DTrace settings.
11995	*
11996	* This updates @a pDbgState and the VMCS execution control fields to reflect
11997	* the necessary VM-exits demanded by DBGF and DTrace.
11998	*
11999	* @param pVCpu The cross context virtual CPU structure.
12000	* @param pVmxTransient The VMX-transient structure. May update
12001	* fUpdatedTscOffsettingAndPreemptTimer.
12002	* @param pDbgState The debug state.
12003	*/
12004	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12005	{
12006	/*
12007	* Take down the dtrace serial number so we can spot changes.
12008	*/
12009	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
12010	ASMCompilerBarrier();
12011
12012	/*
12013	* We'll rebuild most of the middle block of data members (holding the
12014	* current settings) as we go along here, so start by clearing it all.
12015	*/
12016	pDbgState->bmXcptExtra = 0;
12017	pDbgState->fCpe1Extra = 0;
12018	pDbgState->fCpe1Unwanted = 0;
12019	pDbgState->fCpe2Extra = 0;
12020	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
12021	pDbgState->bmExitsToCheck[i] = 0;
12022
12023	/*
12024	* Software interrupts (INT XXh) - no idea how to trigger these...
12025	*/
12026	PVM pVM = pVCpu->CTX_SUFF(pVM);
12027	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
12028	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
12029	{
12030	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12031	}
12032
12033	/*
12034	* INT3 breakpoints - triggered by #BP exceptions.
12035	*/
12036	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
12037	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12038
12039	/*
12040	* Exception bitmap and XCPT events+probes.
12041	*/
12042	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
12043	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
12044	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
12045
12046	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
12047	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
12048	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12049	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
12050	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
12051	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
12052	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
12053	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
12054	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
12055	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
12056	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
12057	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
12058	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
12059	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
12060	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
12061	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
12062	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
12063	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
12064
12065	if (pDbgState->bmXcptExtra)
12066	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12067
12068	/*
12069	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
12070	*
12071	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
12072	* So, when adding/changing/removing please don't forget to update it.
12073	*
12074	* Some of the macros are picking up local variables to save horizontal space,
12075	* (being able to see it in a table is the lesser evil here).
12076	*/
12077	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
12078	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
12079	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
12080	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
12081	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12082	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12083	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12084	} else do { } while (0)
12085	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
12086	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12087	{ \
12088	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
12089	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12090	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12091	} else do { } while (0)
12092	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
12093	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12094	{ \
12095	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
12096	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12097	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12098	} else do { } while (0)
12099	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
12100	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12101	{ \
12102	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
12103	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12104	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12105	} else do { } while (0)
12106
12107	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
12108	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
12109	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
12110	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
12111	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
12112
12113	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
12114	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
12115	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
12116	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
12117	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
12118	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
12119	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
12120	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
12121	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
12122	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
12123	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
12124	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
12125	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
12126	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
12127	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
12128	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
12129	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
12130	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
12131	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
12132	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
12133	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
12134	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
12135	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
12136	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
12137	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
12138	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
12139	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
12140	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
12141	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
12142	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
12143	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
12144	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
12145	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
12146	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
12147	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
12148	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
12149
12150	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
12151	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12152	{
12153	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
12154	\| CPUMCTX_EXTRN_APIC_TPR);
12155	AssertRC(rc);
12156
12157	#if 0 /** @todo fix me */
12158	pDbgState->fClearCr0Mask = true;
12159	pDbgState->fClearCr4Mask = true;
12160	#endif
12161	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
12162	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
12163	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12164	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
12165	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
12166	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
12167	require clearing here and in the loop if we start using it. */
12168	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
12169	}
12170	else
12171	{
12172	if (pDbgState->fClearCr0Mask)
12173	{
12174	pDbgState->fClearCr0Mask = false;
12175	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12176	}
12177	if (pDbgState->fClearCr4Mask)
12178	{
12179	pDbgState->fClearCr4Mask = false;
12180	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12181	}
12182	}
12183	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
12184	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
12185
12186	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
12187	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
12188	{
12189	/** @todo later, need to fix handler as it assumes this won't usually happen. */
12190	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
12191	}
12192	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
12193	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
12194
12195	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
12196	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
12197	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
12198	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
12199	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
12200	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
12201	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
12202	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
12203	#if 0 /** @todo too slow, fix handler. */
12204	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
12205	#endif
12206	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
12207
12208	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
12209	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
12210	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
12211	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
12212	{
12213	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12214	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
12215	}
12216	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12217	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12218	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12219	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12220
12221	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
12222	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
12223	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
12224	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
12225	{
12226	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12227	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
12228	}
12229	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
12230	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
12231	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
12232	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
12233
12234	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
12235	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
12236	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
12237	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
12238	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
12239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
12240	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
12241	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
12242	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
12243	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
12244	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
12245	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
12246	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
12247	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
12248	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
12249	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
12250	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
12251	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
12252	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
12253	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
12254	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
12255	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
12256
12257	#undef IS_EITHER_ENABLED
12258	#undef SET_ONLY_XBM_IF_EITHER_EN
12259	#undef SET_CPE1_XBM_IF_EITHER_EN
12260	#undef SET_CPEU_XBM_IF_EITHER_EN
12261	#undef SET_CPE2_XBM_IF_EITHER_EN
12262
12263	/*
12264	* Sanitize the control stuff.
12265	*/
12266	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
12267	if (pDbgState->fCpe2Extra)
12268	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
12269	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
12270	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
12271	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
12272	{
12273	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
12274	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12275	}
12276
12277	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
12278	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
12279	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
12280	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
12281	}
12282
12283
12284	/**
12285	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
12286	* appropriate.
12287	*
12288	* The caller has checked the VM-exit against the
12289	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
12290	* already, so we don't have to do that either.
12291	*
12292	* @returns Strict VBox status code (i.e. informational status codes too).
12293	* @param pVCpu The cross context virtual CPU structure.
12294	* @param pVmxTransient The VMX-transient structure.
12295	* @param uExitReason The VM-exit reason.
12296	*
12297	* @remarks The name of this function is displayed by dtrace, so keep it short
12298	* and to the point. No longer than 33 chars long, please.
12299	*/
12300	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
12301	{
12302	/*
12303	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
12304	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
12305	*
12306	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
12307	* does. Must add/change/remove both places. Same ordering, please.
12308	*
12309	* Added/removed events must also be reflected in the next section
12310	* where we dispatch dtrace events.
12311	*/
12312	bool fDtrace1 = false;
12313	bool fDtrace2 = false;
12314	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
12315	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
12316	uint32_t uEventArg = 0;
12317	#define SET_EXIT(a_EventSubName) \
12318	do { \
12319	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12320	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12321	} while (0)
12322	#define SET_BOTH(a_EventSubName) \
12323	do { \
12324	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
12325	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12326	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
12327	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12328	} while (0)
12329	switch (uExitReason)
12330	{
12331	case VMX_EXIT_MTF:
12332	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12333
12334	case VMX_EXIT_XCPT_OR_NMI:
12335	{
12336	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12337	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
12338	{
12339	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12340	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
12341	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
12342	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
12343	{
12344	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
12345	{
12346	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12347	uEventArg = pVmxTransient->uExitIntErrorCode;
12348	}
12349	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
12350	switch (enmEvent1)
12351	{
12352	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
12353	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
12354	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
12355	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
12356	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
12357	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
12358	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
12359	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
12360	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
12361	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
12362	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
12363	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
12364	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
12365	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
12366	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
12367	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
12368	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
12369	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
12370	default: break;
12371	}
12372	}
12373	else
12374	AssertFailed();
12375	break;
12376
12377	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
12378	uEventArg = idxVector;
12379	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
12380	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
12381	break;
12382	}
12383	break;
12384	}
12385
12386	case VMX_EXIT_TRIPLE_FAULT:
12387	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
12388	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
12389	break;
12390	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
12391	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
12392	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
12393	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
12394	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
12395
12396	/* Instruction specific VM-exits: */
12397	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
12398	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
12399	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
12400	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
12401	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
12402	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
12403	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
12404	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
12405	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
12406	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
12407	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
12408	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
12409	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
12410	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
12411	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
12412	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
12413	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12414	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12415	case VMX_EXIT_MOV_CRX:
12416	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12417	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12418	SET_BOTH(CRX_READ);
12419	else
12420	SET_BOTH(CRX_WRITE);
12421	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12422	break;
12423	case VMX_EXIT_MOV_DRX:
12424	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12425	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12426	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12427	SET_BOTH(DRX_READ);
12428	else
12429	SET_BOTH(DRX_WRITE);
12430	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12431	break;
12432	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12433	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12434	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12435	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12436	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12437	case VMX_EXIT_GDTR_IDTR_ACCESS:
12438	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12439	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12440	{
12441	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12442	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12443	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12444	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12445	}
12446	break;
12447
12448	case VMX_EXIT_LDTR_TR_ACCESS:
12449	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12450	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12451	{
12452	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12453	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12454	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12455	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12456	}
12457	break;
12458
12459	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12460	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12461	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12462	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12463	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12464	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12465	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12466	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12467	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12468	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12469	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12470
12471	/* Events that aren't relevant at this point. */
12472	case VMX_EXIT_EXT_INT:
12473	case VMX_EXIT_INT_WINDOW:
12474	case VMX_EXIT_NMI_WINDOW:
12475	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12476	case VMX_EXIT_PREEMPT_TIMER:
12477	case VMX_EXIT_IO_INSTR:
12478	break;
12479
12480	/* Errors and unexpected events. */
12481	case VMX_EXIT_INIT_SIGNAL:
12482	case VMX_EXIT_SIPI:
12483	case VMX_EXIT_IO_SMI:
12484	case VMX_EXIT_SMI:
12485	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12486	case VMX_EXIT_ERR_MSR_LOAD:
12487	case VMX_EXIT_ERR_MACHINE_CHECK:
12488	case VMX_EXIT_PML_FULL:
12489	case VMX_EXIT_VIRTUALIZED_EOI:
12490	break;
12491
12492	default:
12493	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12494	break;
12495	}
12496	#undef SET_BOTH
12497	#undef SET_EXIT
12498
12499	/*
12500	* Dtrace tracepoints go first. We do them here at once so we don't
12501	* have to copy the guest state saving and stuff a few dozen times.
12502	* Down side is that we've got to repeat the switch, though this time
12503	* we use enmEvent since the probes are a subset of what DBGF does.
12504	*/
12505	if (fDtrace1 \|\| fDtrace2)
12506	{
12507	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12508	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12509	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12510	switch (enmEvent1)
12511	{
12512	/** @todo consider which extra parameters would be helpful for each probe. */
12513	case DBGFEVENT_END: break;
12514	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12515	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12516	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12517	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12518	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12519	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12520	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12521	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12522	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12523	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12524	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12525	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12526	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12527	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12528	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12529	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12530	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12531	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12532	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12533	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12534	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12535	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12536	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12537	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12538	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12539	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12540	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12541	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12542	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12543	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12544	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12545	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12546	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12547	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12548	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12549	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12550	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12551	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12552	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12553	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12554	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12555	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12556	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12557	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12558	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12559	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12560	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12561	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12562	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12563	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12564	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12565	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12566	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12567	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12568	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12569	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12570	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12571	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12572	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12573	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12574	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12575	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12576	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12577	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12578	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12579	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12580	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12581	}
12582	switch (enmEvent2)
12583	{
12584	/** @todo consider which extra parameters would be helpful for each probe. */
12585	case DBGFEVENT_END: break;
12586	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12587	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12588	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12589	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12590	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12591	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12592	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12593	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12594	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12595	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12596	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12597	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12598	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12599	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12600	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12601	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12602	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12603	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12604	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12605	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12606	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12607	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12608	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12609	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12610	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12611	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12612	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12613	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12614	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12615	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12616	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12617	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12618	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12619	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12620	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12621	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12622	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12623	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12624	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12625	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12626	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12627	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12628	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12629	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12630	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12631	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12632	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12633	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12634	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12635	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12636	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12637	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12638	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12639	}
12640	}
12641
12642	/*
12643	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12644	* the DBGF call will do a full check).
12645	*
12646	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12647	* Note! If we have to events, we prioritize the first, i.e. the instruction
12648	* one, in order to avoid event nesting.
12649	*/
12650	PVM pVM = pVCpu->CTX_SUFF(pVM);
12651	if ( enmEvent1 != DBGFEVENT_END
12652	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12653	{
12654	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12655	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12656	if (rcStrict != VINF_SUCCESS)
12657	return rcStrict;
12658	}
12659	else if ( enmEvent2 != DBGFEVENT_END
12660	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12661	{
12662	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12663	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12664	if (rcStrict != VINF_SUCCESS)
12665	return rcStrict;
12666	}
12667
12668	return VINF_SUCCESS;
12669	}
12670
12671
12672	/**
12673	* Single-stepping VM-exit filtering.
12674	*
12675	* This is preprocessing the VM-exits and deciding whether we've gotten far
12676	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12677	* handling is performed.
12678	*
12679	* @returns Strict VBox status code (i.e. informational status codes too).
12680	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12681	* @param pVmxTransient The VMX-transient structure.
12682	* @param pDbgState The debug state.
12683	*/
12684	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12685	{
12686	/*
12687	* Expensive (saves context) generic dtrace VM-exit probe.
12688	*/
12689	uint32_t const uExitReason = pVmxTransient->uExitReason;
12690	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12691	{ /* more likely */ }
12692	else
12693	{
12694	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12695	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12696	AssertRC(rc);
12697	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12698	}
12699
12700	/*
12701	* Check for host NMI, just to get that out of the way.
12702	*/
12703	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12704	{ /* normally likely */ }
12705	else
12706	{
12707	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12708	AssertRCReturn(rc2, rc2);
12709	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12710	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12711	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12712	}
12713
12714	/*
12715	* Check for single stepping event if we're stepping.
12716	*/
12717	if (pVCpu->hm.s.fSingleInstruction)
12718	{
12719	switch (uExitReason)
12720	{
12721	case VMX_EXIT_MTF:
12722	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12723
12724	/* Various events: */
12725	case VMX_EXIT_XCPT_OR_NMI:
12726	case VMX_EXIT_EXT_INT:
12727	case VMX_EXIT_TRIPLE_FAULT:
12728	case VMX_EXIT_INT_WINDOW:
12729	case VMX_EXIT_NMI_WINDOW:
12730	case VMX_EXIT_TASK_SWITCH:
12731	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12732	case VMX_EXIT_APIC_ACCESS:
12733	case VMX_EXIT_EPT_VIOLATION:
12734	case VMX_EXIT_EPT_MISCONFIG:
12735	case VMX_EXIT_PREEMPT_TIMER:
12736
12737	/* Instruction specific VM-exits: */
12738	case VMX_EXIT_CPUID:
12739	case VMX_EXIT_GETSEC:
12740	case VMX_EXIT_HLT:
12741	case VMX_EXIT_INVD:
12742	case VMX_EXIT_INVLPG:
12743	case VMX_EXIT_RDPMC:
12744	case VMX_EXIT_RDTSC:
12745	case VMX_EXIT_RSM:
12746	case VMX_EXIT_VMCALL:
12747	case VMX_EXIT_VMCLEAR:
12748	case VMX_EXIT_VMLAUNCH:
12749	case VMX_EXIT_VMPTRLD:
12750	case VMX_EXIT_VMPTRST:
12751	case VMX_EXIT_VMREAD:
12752	case VMX_EXIT_VMRESUME:
12753	case VMX_EXIT_VMWRITE:
12754	case VMX_EXIT_VMXOFF:
12755	case VMX_EXIT_VMXON:
12756	case VMX_EXIT_MOV_CRX:
12757	case VMX_EXIT_MOV_DRX:
12758	case VMX_EXIT_IO_INSTR:
12759	case VMX_EXIT_RDMSR:
12760	case VMX_EXIT_WRMSR:
12761	case VMX_EXIT_MWAIT:
12762	case VMX_EXIT_MONITOR:
12763	case VMX_EXIT_PAUSE:
12764	case VMX_EXIT_GDTR_IDTR_ACCESS:
12765	case VMX_EXIT_LDTR_TR_ACCESS:
12766	case VMX_EXIT_INVEPT:
12767	case VMX_EXIT_RDTSCP:
12768	case VMX_EXIT_INVVPID:
12769	case VMX_EXIT_WBINVD:
12770	case VMX_EXIT_XSETBV:
12771	case VMX_EXIT_RDRAND:
12772	case VMX_EXIT_INVPCID:
12773	case VMX_EXIT_VMFUNC:
12774	case VMX_EXIT_RDSEED:
12775	case VMX_EXIT_XSAVES:
12776	case VMX_EXIT_XRSTORS:
12777	{
12778	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12779	AssertRCReturn(rc, rc);
12780	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12781	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12782	return VINF_EM_DBG_STEPPED;
12783	break;
12784	}
12785
12786	/* Errors and unexpected events: */
12787	case VMX_EXIT_INIT_SIGNAL:
12788	case VMX_EXIT_SIPI:
12789	case VMX_EXIT_IO_SMI:
12790	case VMX_EXIT_SMI:
12791	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12792	case VMX_EXIT_ERR_MSR_LOAD:
12793	case VMX_EXIT_ERR_MACHINE_CHECK:
12794	case VMX_EXIT_PML_FULL:
12795	case VMX_EXIT_VIRTUALIZED_EOI:
12796	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12797	break;
12798
12799	default:
12800	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12801	break;
12802	}
12803	}
12804
12805	/*
12806	* Check for debugger event breakpoints and dtrace probes.
12807	*/
12808	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12809	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12810	{
12811	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12812	if (rcStrict != VINF_SUCCESS)
12813	return rcStrict;
12814	}
12815
12816	/*
12817	* Normal processing.
12818	*/
12819	#ifdef HMVMX_USE_FUNCTION_TABLE
12820	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12821	#else
12822	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12823	#endif
12824	}
12825
12826
12827	/**
12828	* Single steps guest code using hardware-assisted VMX.
12829	*
12830	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12831	* but single-stepping through the hypervisor debugger.
12832	*
12833	* @returns Strict VBox status code (i.e. informational status codes too).
12834	* @param pVCpu The cross context virtual CPU structure.
12835	* @param pcLoops Pointer to the number of executed loops.
12836	*
12837	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12838	*/
12839	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12840	{
12841	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12842	Assert(pcLoops);
12843	Assert(*pcLoops <= cMaxResumeLoops);
12844
12845	VMXTRANSIENT VmxTransient;
12846	RT_ZERO(VmxTransient);
12847	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12848
12849	/* Set HMCPU indicators. */
12850	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12851	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12852	pVCpu->hm.s.fDebugWantRdTscExit = false;
12853	pVCpu->hm.s.fUsingDebugLoop = true;
12854
12855	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12856	VMXRUNDBGSTATE DbgState;
12857	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12858	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12859
12860	/*
12861	* The loop.
12862	*/
12863	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12864	for (;;)
12865	{
12866	Assert(!HMR0SuspendPending());
12867	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12868	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12869	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12870
12871	/* Set up VM-execution controls the next two can respond to. */
12872	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12873
12874	/*
12875	* Preparatory work for running guest code, this may force us to
12876	* return to ring-3.
12877	*
12878	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12879	*/
12880	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12881	if (rcStrict != VINF_SUCCESS)
12882	break;
12883
12884	/* Interrupts are disabled at this point! */
12885	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12886
12887	/* Override any obnoxious code in the above two calls. */
12888	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12889
12890	/*
12891	* Finally execute the guest.
12892	*/
12893	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12894
12895	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12896	/* Interrupts are re-enabled at this point! */
12897
12898	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12899	if (RT_SUCCESS(rcRun))
12900	{ /* very likely */ }
12901	else
12902	{
12903	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12904	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12905	return rcRun;
12906	}
12907
12908	/* Profile the VM-exit. */
12909	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12911	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12912	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12913	HMVMX_START_EXIT_DISPATCH_PROF();
12914
12915	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12916
12917	/*
12918	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12919	*/
12920	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12921	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12922	if (rcStrict != VINF_SUCCESS)
12923	break;
12924	if (++(*pcLoops) > cMaxResumeLoops)
12925	{
12926	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12927	rcStrict = VINF_EM_RAW_INTERRUPT;
12928	break;
12929	}
12930
12931	/*
12932	* Stepping: Did the RIP change, if so, consider it a single step.
12933	* Otherwise, make sure one of the TFs gets set.
12934	*/
12935	if (fStepping)
12936	{
12937	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12938	AssertRC(rc);
12939	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12940	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12941	{
12942	rcStrict = VINF_EM_DBG_STEPPED;
12943	break;
12944	}
12945	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12946	}
12947
12948	/*
12949	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12950	*/
12951	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12952	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12953	}
12954
12955	/*
12956	* Clear the X86_EFL_TF if necessary.
12957	*/
12958	if (pVCpu->hm.s.fClearTrapFlag)
12959	{
12960	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12961	AssertRC(rc);
12962	pVCpu->hm.s.fClearTrapFlag = false;
12963	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12964	}
12965	/** @todo there seems to be issues with the resume flag when the monitor trap
12966	* flag is pending without being used. Seen early in bios init when
12967	* accessing APIC page in protected mode. */
12968
12969	/*
12970	* Restore VM-exit control settings as we may not re-enter this function the
12971	* next time around.
12972	*/
12973	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12974
12975	/* Restore HMCPU indicators. */
12976	pVCpu->hm.s.fUsingDebugLoop = false;
12977	pVCpu->hm.s.fDebugWantRdTscExit = false;
12978	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12979
12980	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12981	return rcStrict;
12982	}
12983
12984
12985	/** @} */
12986
12987
12988	/**
12989	* Checks if any expensive dtrace probes are enabled and we should go to the
12990	* debug loop.
12991	*
12992	* @returns true if we should use debug loop, false if not.
12993	*/
12994	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12995	{
12996	/* It's probably faster to OR the raw 32-bit counter variables together.
12997	Since the variables are in an array and the probes are next to one
12998	another (more or less), we have good locality. So, better read
12999	eight-nine cache lines ever time and only have one conditional, than
13000	128+ conditionals, right? */
13001	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
13002	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
13003	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
13004	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
13005	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
13006	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
13007	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
13008	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
13009	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
13010	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
13011	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
13012	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
13013	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
13014	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
13015	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
13016	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
13017	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
13018	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
13019	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
13020	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
13021	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
13022	) != 0
13023	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
13024	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
13025	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
13026	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
13027	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
13028	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
13029	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
13030	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
13031	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
13032	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
13033	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
13034	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
13035	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
13036	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
13037	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
13038	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
13039	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
13040	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
13041	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
13042	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
13043	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
13044	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
13045	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
13046	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
13047	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
13048	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
13049	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
13050	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
13051	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
13052	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
13053	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
13054	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
13055	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
13056	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
13057	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
13058	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
13059	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
13060	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
13061	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
13062	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
13063	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
13064	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
13065	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
13066	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
13067	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
13068	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
13069	) != 0
13070	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
13071	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
13072	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
13073	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
13074	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
13075	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
13076	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
13077	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
13078	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
13079	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
13080	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
13081	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
13082	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
13083	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
13084	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
13085	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
13086	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
13087	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
13088	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
13089	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
13090	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
13091	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
13092	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
13093	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
13094	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
13095	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
13096	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
13097	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
13098	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
13099	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
13100	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
13101	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
13102	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
13103	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
13104	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
13105	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
13106	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
13107	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
13108	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
13109	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
13110	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
13111	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
13112	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
13113	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
13114	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
13115	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
13116	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
13117	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
13118	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
13119	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
13120	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
13121	) != 0;
13122	}
13123
13124
13125	/**
13126	* Runs the guest using hardware-assisted VMX.
13127	*
13128	* @returns Strict VBox status code (i.e. informational status codes too).
13129	* @param pVCpu The cross context virtual CPU structure.
13130	*/
13131	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
13132	{
13133	AssertPtr(pVCpu);
13134	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13135	Assert(VMMRZCallRing3IsEnabled(pVCpu));
13136	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13137	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
13138
13139	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
13140
13141	VBOXSTRICTRC rcStrict;
13142	uint32_t cLoops = 0;
13143	for (;;)
13144	{
13145	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13146	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
13147	#else
13148	bool const fInNestedGuestMode = false;
13149	#endif
13150	if (!fInNestedGuestMode)
13151	{
13152	if ( !pVCpu->hm.s.fUseDebugLoop
13153	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
13154	&& !DBGFIsStepping(pVCpu)
13155	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
13156	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
13157	else
13158	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
13159	}
13160	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13161	else
13162	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
13163
13164	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
13165	{
13166	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
13167	continue;
13168	}
13169	if (rcStrict == VINF_VMX_VMEXIT)
13170	{
13171	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
13172	continue;
13173	}
13174	#endif
13175	break;
13176	}
13177
13178	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
13179	switch (rcLoop)
13180	{
13181	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
13182	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
13183	}
13184
13185	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
13186	if (RT_FAILURE(rc2))
13187	{
13188	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
13189	rcStrict = rc2;
13190	}
13191	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13192	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
13193	return rcStrict;
13194	}
13195
13196
13197	#ifndef HMVMX_USE_FUNCTION_TABLE
13198	/**
13199	* Handles a guest VM-exit from hardware-assisted VMX execution.
13200	*
13201	* @returns Strict VBox status code (i.e. informational status codes too).
13202	* @param pVCpu The cross context virtual CPU structure.
13203	* @param pVmxTransient The VMX-transient structure.
13204	*/
13205	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13206	{
13207	#ifdef DEBUG_ramshankar
13208	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
13209	do { \
13210	if (a_fSave != 0) \
13211	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
13212	VBOXSTRICTRC rcStrict = a_CallExpr; \
13213	if (a_fSave != 0) \
13214	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
13215	return rcStrict; \
13216	} while (0)
13217	#else
13218	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
13219	#endif
13220	uint32_t const uExitReason = pVmxTransient->uExitReason;
13221	switch (uExitReason)
13222	{
13223	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
13224	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
13225	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
13226	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
13227	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
13228	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
13229	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
13230	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
13231	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
13232	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
13233	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
13234	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
13235	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
13236	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
13237	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
13238	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
13239	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
13240	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
13241	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
13242	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
13243	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
13244	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
13245	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
13246	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
13247	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
13248	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
13249	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
13250	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
13251	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
13252	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
13253	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13254	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
13255	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
13256	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
13257	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
13258	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
13259	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
13260	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
13261	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
13262	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
13263	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
13264	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
13265	#else
13266	case VMX_EXIT_VMCLEAR:
13267	case VMX_EXIT_VMLAUNCH:
13268	case VMX_EXIT_VMPTRLD:
13269	case VMX_EXIT_VMPTRST:
13270	case VMX_EXIT_VMREAD:
13271	case VMX_EXIT_VMRESUME:
13272	case VMX_EXIT_VMWRITE:
13273	case VMX_EXIT_VMXOFF:
13274	case VMX_EXIT_VMXON:
13275	case VMX_EXIT_INVVPID:
13276	case VMX_EXIT_INVEPT:
13277	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
13278	#endif
13279
13280	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
13281	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
13282	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
13283
13284	case VMX_EXIT_INIT_SIGNAL:
13285	case VMX_EXIT_SIPI:
13286	case VMX_EXIT_IO_SMI:
13287	case VMX_EXIT_SMI:
13288	case VMX_EXIT_ERR_MSR_LOAD:
13289	case VMX_EXIT_ERR_MACHINE_CHECK:
13290	case VMX_EXIT_PML_FULL:
13291	case VMX_EXIT_VIRTUALIZED_EOI:
13292	case VMX_EXIT_GDTR_IDTR_ACCESS:
13293	case VMX_EXIT_LDTR_TR_ACCESS:
13294	case VMX_EXIT_APIC_WRITE:
13295	case VMX_EXIT_RDRAND:
13296	case VMX_EXIT_RSM:
13297	case VMX_EXIT_VMFUNC:
13298	case VMX_EXIT_ENCLS:
13299	case VMX_EXIT_RDSEED:
13300	case VMX_EXIT_XSAVES:
13301	case VMX_EXIT_XRSTORS:
13302	case VMX_EXIT_UMWAIT:
13303	case VMX_EXIT_TPAUSE:
13304	default:
13305	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13306	}
13307	#undef VMEXIT_CALL_RET
13308	}
13309	#endif /* !HMVMX_USE_FUNCTION_TABLE */
13310
13311
13312	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13313	/**
13314	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
13315	*
13316	* @returns Strict VBox status code (i.e. informational status codes too).
13317	* @param pVCpu The cross context virtual CPU structure.
13318	* @param pVmxTransient The VMX-transient structure.
13319	*/
13320	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13321	{
13322	uint32_t const uExitReason = pVmxTransient->uExitReason;
13323	switch (uExitReason)
13324	{
13325	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
13326	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
13327	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
13328	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
13329	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
13330
13331	/*
13332	* We shouldn't direct host physical interrupts to the nested-guest.
13333	*/
13334	case VMX_EXIT_EXT_INT:
13335	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
13336
13337	/*
13338	* Instructions that cause VM-exits unconditionally or the condition is
13339	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13340	* happens, it's guaranteed to be a nested-guest VM-exit).
13341	*
13342	* - Provides VM-exit instruction length ONLY.
13343	*/
13344	case VMX_EXIT_CPUID: /* Unconditional. */
13345	case VMX_EXIT_VMCALL:
13346	case VMX_EXIT_GETSEC:
13347	case VMX_EXIT_INVD:
13348	case VMX_EXIT_XSETBV:
13349	case VMX_EXIT_VMLAUNCH:
13350	case VMX_EXIT_VMRESUME:
13351	case VMX_EXIT_VMXOFF:
13352	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
13353	case VMX_EXIT_VMFUNC:
13354	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
13355
13356	/*
13357	* Instructions that cause VM-exits unconditionally or the condition is
13358	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13359	* happens, it's guaranteed to be a nested-guest VM-exit).
13360	*
13361	* - Provides VM-exit instruction length.
13362	* - Provides VM-exit information.
13363	* - Optionally provides Exit qualification.
13364	*
13365	* Since Exit qualification is 0 for all VM-exits where it is not
13366	* applicable, reading and passing it to the guest should produce
13367	* defined behavior.
13368	*
13369	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
13370	*/
13371	case VMX_EXIT_INVEPT: /* Unconditional. */
13372	case VMX_EXIT_INVVPID:
13373	case VMX_EXIT_VMCLEAR:
13374	case VMX_EXIT_VMPTRLD:
13375	case VMX_EXIT_VMPTRST:
13376	case VMX_EXIT_VMXON:
13377	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
13378	case VMX_EXIT_LDTR_TR_ACCESS:
13379	case VMX_EXIT_RDRAND:
13380	case VMX_EXIT_RDSEED:
13381	case VMX_EXIT_XSAVES:
13382	case VMX_EXIT_XRSTORS:
13383	case VMX_EXIT_UMWAIT:
13384	case VMX_EXIT_TPAUSE:
13385	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
13386
13387	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
13388	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
13389	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
13390	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
13391	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
13392	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
13393	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
13394	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
13395	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
13396	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
13397	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
13398	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
13399	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
13400	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
13401	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
13402	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
13403	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
13404	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
13405	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
13406
13407	case VMX_EXIT_PREEMPT_TIMER:
13408	{
13409	/** @todo NSTVMX: Preempt timer. */
13410	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
13411	}
13412
13413	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
13414	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
13415
13416	case VMX_EXIT_VMREAD:
13417	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13418
13419	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13420	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13421
13422	case VMX_EXIT_INIT_SIGNAL:
13423	case VMX_EXIT_SIPI:
13424	case VMX_EXIT_IO_SMI:
13425	case VMX_EXIT_SMI:
13426	case VMX_EXIT_ERR_MSR_LOAD:
13427	case VMX_EXIT_ERR_MACHINE_CHECK:
13428	case VMX_EXIT_PML_FULL:
13429	case VMX_EXIT_RSM:
13430	default:
13431	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13432	}
13433	}
13434	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13435
13436
13437	/** @name VM-exit helpers.
13438	* @{
13439	*/
13440	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13441	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13442	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13443
13444	/** Macro for VM-exits called unexpectedly. */
13445	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
13446	do { \
13447	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
13448	return VERR_VMX_UNEXPECTED_EXIT; \
13449	} while (0)
13450
13451	#ifdef VBOX_STRICT
13452	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13453	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13454	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13455
13456	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13457	do { \
13458	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13459	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13460	} while (0)
13461
13462	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13463	do { \
13464	AssertPtr((a_pVCpu)); \
13465	AssertPtr((a_pVmxTransient)); \
13466	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13467	Assert((a_pVmxTransient)->pVmcsInfo); \
13468	Assert(ASMIntAreEnabled()); \
13469	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13470	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13471	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
13472	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13473	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13474	HMVMX_ASSERT_PREEMPT_CPUID(); \
13475	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13476	} while (0)
13477
13478	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13479	do { \
13480	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13481	Assert((a_pVmxTransient)->fIsNestedGuest); \
13482	} while (0)
13483
13484	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13485	do { \
13486	Log4Func(("\n")); \
13487	} while (0)
13488	#else
13489	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13490	do { \
13491	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13492	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13493	} while (0)
13494
13495	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13496	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13497
13498	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13499	#endif
13500
13501	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13502	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13503	* guests that attempted to execute a VMX instruction. */
13504	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13505	do \
13506	{ \
13507	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13508	if (rcStrictTmp == VINF_SUCCESS) \
13509	{ /* likely */ } \
13510	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13511	{ \
13512	Assert((a_pVCpu)->hm.s.Event.fPending); \
13513	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13514	return VINF_SUCCESS; \
13515	} \
13516	else \
13517	{ \
13518	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13519	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13520	} \
13521	} while (0)
13522
13523	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13524	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13525	do \
13526	{ \
13527	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13528	(a_pGCPtrEffAddr)); \
13529	if (rcStrictTmp == VINF_SUCCESS) \
13530	{ /* likely */ } \
13531	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13532	{ \
13533	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13534	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13535	NOREF(uXcptTmp); \
13536	return VINF_SUCCESS; \
13537	} \
13538	else \
13539	{ \
13540	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13541	return rcStrictTmp; \
13542	} \
13543	} while (0)
13544	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13545
13546
13547	/**
13548	* Advances the guest RIP by the specified number of bytes.
13549	*
13550	* @param pVCpu The cross context virtual CPU structure.
13551	* @param cbInstr Number of bytes to advance the RIP by.
13552	*
13553	* @remarks No-long-jump zone!!!
13554	*/
13555	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13556	{
13557	/* Advance the RIP. */
13558	pVCpu->cpum.GstCtx.rip += cbInstr;
13559	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13560
13561	/* Update interrupt inhibition. */
13562	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13563	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13564	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13565	}
13566
13567
13568	/**
13569	* Advances the guest RIP after reading it from the VMCS.
13570	*
13571	* @returns VBox status code, no informational status codes.
13572	* @param pVCpu The cross context virtual CPU structure.
13573	* @param pVmxTransient The VMX-transient structure.
13574	*
13575	* @remarks No-long-jump zone!!!
13576	*/
13577	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13578	{
13579	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13580	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13581	AssertRCReturn(rc, rc);
13582
13583	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13584	return VINF_SUCCESS;
13585	}
13586
13587
13588	/**
13589	* Handle a condition that occurred while delivering an event through the guest
13590	* IDT.
13591	*
13592	* @returns Strict VBox status code (i.e. informational status codes too).
13593	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13594	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13595	* to continue execution of the guest which will delivery the \#DF.
13596	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13597	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13598	*
13599	* @param pVCpu The cross context virtual CPU structure.
13600	* @param pVmxTransient The VMX-transient structure.
13601	*
13602	* @remarks No-long-jump zone!!!
13603	*/
13604	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13605	{
13606	/* Read the IDT vectoring info. and VM-exit interruption info. */
13607	{
13608	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13609	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13610	AssertRCReturn(rc, rc);
13611	}
13612
13613	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13614	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13615	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13616	{
13617	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13618	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13619
13620	/*
13621	* If the event was a software interrupt (generated with INT n) or a software exception
13622	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13623	* can handle the VM-exit and continue guest execution which will re-execute the
13624	* instruction rather than re-injecting the exception, as that can cause premature
13625	* trips to ring-3 before injection and involve TRPM which currently has no way of
13626	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13627	* the problem).
13628	*/
13629	IEMXCPTRAISE enmRaise;
13630	IEMXCPTRAISEINFO fRaiseInfo;
13631	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13632	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13633	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13634	{
13635	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13636	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13637	}
13638	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13639	{
13640	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13641	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13642	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13643
13644	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13645	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13646	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13647
13648	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13649
13650	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13651	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13652	{
13653	pVmxTransient->fVectoringPF = true;
13654	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13655	}
13656	}
13657	else
13658	{
13659	/*
13660	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13661	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13662	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13663	*/
13664	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13665	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13666	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13667	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13668	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13669	}
13670
13671	/*
13672	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13673	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13674	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13675	* subsequent VM-entry would fail, see @bugref{7445}.
13676	*
13677	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13678	*/
13679	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13680	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13681	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13682	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13683	&& CPUMIsGuestNmiBlocking(pVCpu))
13684	{
13685	CPUMSetGuestNmiBlocking(pVCpu, false);
13686	}
13687
13688	switch (enmRaise)
13689	{
13690	case IEMXCPTRAISE_CURRENT_XCPT:
13691	{
13692	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13693	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13694	Assert(rcStrict == VINF_SUCCESS);
13695	break;
13696	}
13697
13698	case IEMXCPTRAISE_PREV_EVENT:
13699	{
13700	uint32_t u32ErrCode;
13701	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13702	{
13703	int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13704	AssertRCReturn(rc, rc);
13705	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13706	}
13707	else
13708	u32ErrCode = 0;
13709
13710	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13711	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13712	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13713	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13714
13715	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13716	pVCpu->hm.s.Event.u32ErrCode));
13717	Assert(rcStrict == VINF_SUCCESS);
13718	break;
13719	}
13720
13721	case IEMXCPTRAISE_REEXEC_INSTR:
13722	Assert(rcStrict == VINF_SUCCESS);
13723	break;
13724
13725	case IEMXCPTRAISE_DOUBLE_FAULT:
13726	{
13727	/*
13728	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13729	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13730	*/
13731	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13732	{
13733	pVmxTransient->fVectoringDoublePF = true;
13734	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13735	pVCpu->cpum.GstCtx.cr2));
13736	rcStrict = VINF_SUCCESS;
13737	}
13738	else
13739	{
13740	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13741	hmR0VmxSetPendingXcptDF(pVCpu);
13742	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13743	uIdtVector, uExitVector));
13744	rcStrict = VINF_HM_DOUBLE_FAULT;
13745	}
13746	break;
13747	}
13748
13749	case IEMXCPTRAISE_TRIPLE_FAULT:
13750	{
13751	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13752	rcStrict = VINF_EM_RESET;
13753	break;
13754	}
13755
13756	case IEMXCPTRAISE_CPU_HANG:
13757	{
13758	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13759	rcStrict = VERR_EM_GUEST_CPU_HANG;
13760	break;
13761	}
13762
13763	default:
13764	{
13765	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13766	rcStrict = VERR_VMX_IPE_2;
13767	break;
13768	}
13769	}
13770	}
13771	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13772	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13773	&& uExitVector != X86_XCPT_DF
13774	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13775	{
13776	Assert(!VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo));
13777
13778	/*
13779	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13780	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13781	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13782	*/
13783	CPUMSetGuestNmiBlocking(pVCpu, true);
13784	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13785	}
13786
13787	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13788	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13789	return rcStrict;
13790	}
13791
13792
13793	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13794	/**
13795	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13796	* guest attempting to execute a VMX instruction.
13797	*
13798	* @returns Strict VBox status code (i.e. informational status codes too).
13799	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13800	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13801	*
13802	* @param pVCpu The cross context virtual CPU structure.
13803	* @param uExitReason The VM-exit reason.
13804	*
13805	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13806	* @remarks No-long-jump zone!!!
13807	*/
13808	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
13809	{
13810	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13811	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13812
13813	/*
13814	* The physical CPU would have already checked the CPU mode/code segment.
13815	* We shall just assert here for paranoia.
13816	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13817	*/
13818	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13819	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13820	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13821
13822	if (uExitReason == VMX_EXIT_VMXON)
13823	{
13824	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13825
13826	/*
13827	* We check CR4.VMXE because it is required to be always set while in VMX operation
13828	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13829	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13830	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13831	*/
13832	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13833	{
13834	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13835	hmR0VmxSetPendingXcptUD(pVCpu);
13836	return VINF_HM_PENDING_XCPT;
13837	}
13838	}
13839	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13840	{
13841	/*
13842	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13843	* (other than VMXON), we need to raise a #UD.
13844	*/
13845	Log4Func(("Not in VMX root mode -> #UD\n"));
13846	hmR0VmxSetPendingXcptUD(pVCpu);
13847	return VINF_HM_PENDING_XCPT;
13848	}
13849
13850	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13851	return VINF_SUCCESS;
13852	}
13853
13854	/**
13855	* Decodes the memory operand of an instruction that caused a VM-exit.
13856	*
13857	* The Exit qualification field provides the displacement field for memory
13858	* operand instructions, if any.
13859	*
13860	* @returns Strict VBox status code (i.e. informational status codes too).
13861	* @retval VINF_SUCCESS if the operand was successfully decoded.
13862	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13863	* operand.
13864	* @param pVCpu The cross context virtual CPU structure.
13865	* @param uExitInstrInfo The VM-exit instruction information field.
13866	* @param enmMemAccess The memory operand's access type (read or write).
13867	* @param GCPtrDisp The instruction displacement field, if any. For
13868	* RIP-relative addressing pass RIP + displacement here.
13869	* @param pGCPtrMem Where to store the effective destination memory address.
13870	*
13871	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13872	* virtual-8086 mode hence skips those checks while verifying if the
13873	* segment is valid.
13874	*/
13875	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13876	PRTGCPTR pGCPtrMem)
13877	{
13878	Assert(pGCPtrMem);
13879	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13880	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13881	\| CPUMCTX_EXTRN_CR0);
13882
13883	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13884	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13885	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13886
13887	VMXEXITINSTRINFO ExitInstrInfo;
13888	ExitInstrInfo.u = uExitInstrInfo;
13889	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13890	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13891	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13892	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13893	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13894	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13895	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13896	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13897	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13898
13899	/*
13900	* Validate instruction information.
13901	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13902	*/
13903	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13904	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13905	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13906	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13907	AssertLogRelMsgReturn(fIsMemOperand,
13908	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13909
13910	/*
13911	* Compute the complete effective address.
13912	*
13913	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13914	* See AMD spec. 4.5.2 "Segment Registers".
13915	*/
13916	RTGCPTR GCPtrMem = GCPtrDisp;
13917	if (fBaseRegValid)
13918	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13919	if (fIdxRegValid)
13920	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13921
13922	RTGCPTR const GCPtrOff = GCPtrMem;
13923	if ( !fIsLongMode
13924	\|\| iSegReg >= X86_SREG_FS)
13925	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13926	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13927
13928	/*
13929	* Validate effective address.
13930	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13931	*/
13932	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13933	Assert(cbAccess > 0);
13934	if (fIsLongMode)
13935	{
13936	if (X86_IS_CANONICAL(GCPtrMem))
13937	{
13938	*pGCPtrMem = GCPtrMem;
13939	return VINF_SUCCESS;
13940	}
13941
13942	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13943	* "Data Limit Checks in 64-bit Mode". */
13944	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13945	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13946	return VINF_HM_PENDING_XCPT;
13947	}
13948
13949	/*
13950	* This is a watered down version of iemMemApplySegment().
13951	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13952	* and segment CPL/DPL checks are skipped.
13953	*/
13954	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13955	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13956	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13957
13958	/* Check if the segment is present and usable. */
13959	if ( pSel->Attr.n.u1Present
13960	&& !pSel->Attr.n.u1Unusable)
13961	{
13962	Assert(pSel->Attr.n.u1DescType);
13963	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13964	{
13965	/* Check permissions for the data segment. */
13966	if ( enmMemAccess == VMXMEMACCESS_WRITE
13967	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13968	{
13969	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13970	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13971	return VINF_HM_PENDING_XCPT;
13972	}
13973
13974	/* Check limits if it's a normal data segment. */
13975	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13976	{
13977	if ( GCPtrFirst32 > pSel->u32Limit
13978	\|\| GCPtrLast32 > pSel->u32Limit)
13979	{
13980	Log4Func(("Data segment limit exceeded. "
13981	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13982	GCPtrLast32, pSel->u32Limit));
13983	if (iSegReg == X86_SREG_SS)
13984	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13985	else
13986	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13987	return VINF_HM_PENDING_XCPT;
13988	}
13989	}
13990	else
13991	{
13992	/* Check limits if it's an expand-down data segment.
13993	Note! The upper boundary is defined by the B bit, not the G bit! */
13994	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13995	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13996	{
13997	Log4Func(("Expand-down data segment limit exceeded. "
13998	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13999	GCPtrLast32, pSel->u32Limit));
14000	if (iSegReg == X86_SREG_SS)
14001	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14002	else
14003	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14004	return VINF_HM_PENDING_XCPT;
14005	}
14006	}
14007	}
14008	else
14009	{
14010	/* Check permissions for the code segment. */
14011	if ( enmMemAccess == VMXMEMACCESS_WRITE
14012	\|\| ( enmMemAccess == VMXMEMACCESS_READ
14013	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
14014	{
14015	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
14016	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
14017	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14018	return VINF_HM_PENDING_XCPT;
14019	}
14020
14021	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
14022	if ( GCPtrFirst32 > pSel->u32Limit
14023	\|\| GCPtrLast32 > pSel->u32Limit)
14024	{
14025	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
14026	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
14027	if (iSegReg == X86_SREG_SS)
14028	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14029	else
14030	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14031	return VINF_HM_PENDING_XCPT;
14032	}
14033	}
14034	}
14035	else
14036	{
14037	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14038	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14039	return VINF_HM_PENDING_XCPT;
14040	}
14041
14042	*pGCPtrMem = GCPtrMem;
14043	return VINF_SUCCESS;
14044	}
14045	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
14046
14047
14048	/**
14049	* VM-exit helper for LMSW.
14050	*/
14051	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
14052	{
14053	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14054	AssertRCReturn(rc, rc);
14055
14056	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
14057	AssertMsg( rcStrict == VINF_SUCCESS
14058	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14059
14060	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14061	if (rcStrict == VINF_IEM_RAISED_XCPT)
14062	{
14063	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14064	rcStrict = VINF_SUCCESS;
14065	}
14066
14067	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14068	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14069	return rcStrict;
14070	}
14071
14072
14073	/**
14074	* VM-exit helper for CLTS.
14075	*/
14076	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
14077	{
14078	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14079	AssertRCReturn(rc, rc);
14080
14081	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
14082	AssertMsg( rcStrict == VINF_SUCCESS
14083	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14084
14085	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14086	if (rcStrict == VINF_IEM_RAISED_XCPT)
14087	{
14088	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14089	rcStrict = VINF_SUCCESS;
14090	}
14091
14092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14093	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14094	return rcStrict;
14095	}
14096
14097
14098	/**
14099	* VM-exit helper for MOV from CRx (CRx read).
14100	*/
14101	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14102	{
14103	Assert(iCrReg < 16);
14104	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14105
14106	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14107	AssertRCReturn(rc, rc);
14108
14109	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
14110	AssertMsg( rcStrict == VINF_SUCCESS
14111	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14112
14113	if (iGReg == X86_GREG_xSP)
14114	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14115	else
14116	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14117	#ifdef VBOX_WITH_STATISTICS
14118	switch (iCrReg)
14119	{
14120	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14121	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14122	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14123	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14124	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14125	}
14126	#endif
14127	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
14128	return rcStrict;
14129	}
14130
14131
14132	/**
14133	* VM-exit helper for MOV to CRx (CRx write).
14134	*/
14135	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14136	{
14137	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14138	AssertRCReturn(rc, rc);
14139
14140	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
14141	AssertMsg( rcStrict == VINF_SUCCESS
14142	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14143	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14144
14145	switch (iCrReg)
14146	{
14147	case 0:
14148	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14149	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14150	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14151	break;
14152
14153	case 2:
14154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14155	/* Nothing to do here, CR2 it's not part of the VMCS. */
14156	break;
14157
14158	case 3:
14159	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14160	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14161	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14162	break;
14163
14164	case 4:
14165	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14166	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14167	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14168	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14169	break;
14170
14171	case 8:
14172	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14173	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14175	break;
14176
14177	default:
14178	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
14179	break;
14180	}
14181
14182	if (rcStrict == VINF_IEM_RAISED_XCPT)
14183	{
14184	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14185	rcStrict = VINF_SUCCESS;
14186	}
14187	return rcStrict;
14188	}
14189
14190
14191	/**
14192	* VM-exit exception handler for \#PF (Page-fault exception).
14193	*/
14194	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14195	{
14196	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14197	PVM pVM = pVCpu->CTX_SUFF(pVM);
14198	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14199	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14200	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14201	AssertRCReturn(rc, rc);
14202
14203	if (!pVM->hm.s.fNestedPaging)
14204	{ /* likely */ }
14205	else
14206	{
14207	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
14208	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
14209	#endif
14210	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
14211	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
14212	{
14213	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14214	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
14215	}
14216	else
14217	{
14218	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14219	hmR0VmxSetPendingXcptDF(pVCpu);
14220	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
14221	}
14222	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14223	return rc;
14224	}
14225
14226	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
14227	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
14228	if (pVmxTransient->fVectoringPF)
14229	{
14230	Assert(pVCpu->hm.s.Event.fPending);
14231	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14232	}
14233
14234	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14235	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14236	AssertRCReturn(rc, rc);
14237
14238	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
14239	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
14240
14241	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
14242	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
14243
14244	Log4Func(("#PF: rc=%Rrc\n", rc));
14245	if (rc == VINF_SUCCESS)
14246	{
14247	/*
14248	* This is typically a shadow page table sync or a MMIO instruction. But we may have
14249	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
14250	*/
14251	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14252	TRPMResetTrap(pVCpu);
14253	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
14254	return rc;
14255	}
14256
14257	if (rc == VINF_EM_RAW_GUEST_TRAP)
14258	{
14259	if (!pVmxTransient->fVectoringDoublePF)
14260	{
14261	/* It's a guest page fault and needs to be reflected to the guest. */
14262	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
14263	TRPMResetTrap(pVCpu);
14264	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
14265	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14266	uGstErrorCode, pVmxTransient->uExitQual);
14267	}
14268	else
14269	{
14270	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14271	TRPMResetTrap(pVCpu);
14272	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14273	hmR0VmxSetPendingXcptDF(pVCpu);
14274	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
14275	}
14276
14277	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14278	return VINF_SUCCESS;
14279	}
14280
14281	TRPMResetTrap(pVCpu);
14282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14283	return rc;
14284	}
14285
14286
14287	/**
14288	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14289	*/
14290	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14291	{
14292	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14293	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14294
14295	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14296	AssertRCReturn(rc, rc);
14297
14298	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14299	{
14300	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14301	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14302
14303	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14304	* provides VM-exit instruction length. If this causes problem later,
14305	* disassemble the instruction like it's done on AMD-V. */
14306	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14307	AssertRCReturn(rc2, rc2);
14308	return rc;
14309	}
14310
14311	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14312	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14313	return rc;
14314	}
14315
14316
14317	/**
14318	* VM-exit exception handler for \#BP (Breakpoint exception).
14319	*/
14320	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14321	{
14322	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14323	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14324
14325	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14326	AssertRCReturn(rc, rc);
14327
14328	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14329	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14330	if (rc == VINF_EM_RAW_GUEST_TRAP)
14331	{
14332	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14333	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14334	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14335	AssertRCReturn(rc, rc);
14336
14337	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14338	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14339	}
14340
14341	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14342	return rc;
14343	}
14344
14345
14346	/**
14347	* VM-exit exception handler for \#AC (alignment check exception).
14348	*/
14349	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14350	{
14351	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14352	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
14353
14354	/*
14355	* Re-inject it. We'll detect any nesting before getting here.
14356	*/
14357	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14358	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14359	AssertRCReturn(rc, rc);
14360	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14361
14362	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14363	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14364	return VINF_SUCCESS;
14365	}
14366
14367
14368	/**
14369	* VM-exit exception handler for \#DB (Debug exception).
14370	*/
14371	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14372	{
14373	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14374	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14375
14376	/*
14377	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
14378	*/
14379	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14380
14381	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14382	uint64_t const uDR6 = X86_DR6_INIT_VAL
14383	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14384	\| X86_DR6_BD \| X86_DR6_BS));
14385
14386	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14387	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14388	Log6Func(("rc=%Rrc\n", rc));
14389	if (rc == VINF_EM_RAW_GUEST_TRAP)
14390	{
14391	/*
14392	* The exception was for the guest. Update DR6, DR7.GD and
14393	* IA32_DEBUGCTL.LBR before forwarding it.
14394	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14395	*/
14396	VMMRZCallRing3Disable(pVCpu);
14397	HM_DISABLE_PREEMPT(pVCpu);
14398
14399	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14400	pCtx->dr[6] \|= uDR6;
14401	if (CPUMIsGuestDebugStateActive(pVCpu))
14402	ASMSetDR6(pCtx->dr[6]);
14403
14404	HM_RESTORE_PREEMPT();
14405	VMMRZCallRing3Enable(pVCpu);
14406
14407	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14408	AssertRCReturn(rc, rc);
14409
14410	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14411	pCtx->dr[7] &= ~X86_DR7_GD;
14412
14413	/* Paranoia. */
14414	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14415	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14416
14417	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14418	AssertRCReturn(rc, rc);
14419
14420	/*
14421	* Raise #DB in the guest.
14422	*
14423	* It is important to reflect exactly what the VM-exit gave us (preserving the
14424	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14425	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14426	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14427	*
14428	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14429	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14430	*/
14431	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14432	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14433	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14434	AssertRCReturn(rc, rc);
14435	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14436	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14437	return VINF_SUCCESS;
14438	}
14439
14440	/*
14441	* Not a guest trap, must be a hypervisor related debug event then.
14442	* Update DR6 in case someone is interested in it.
14443	*/
14444	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14445	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14446	CPUMSetHyperDR6(pVCpu, uDR6);
14447
14448	return rc;
14449	}
14450
14451
14452	/**
14453	* Hacks its way around the lovely mesa driver's backdoor accesses.
14454	*
14455	* @sa hmR0SvmHandleMesaDrvGp.
14456	*/
14457	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14458	{
14459	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14460	RT_NOREF(pCtx);
14461
14462	/* For now we'll just skip the instruction. */
14463	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14464	}
14465
14466
14467	/**
14468	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14469	* backdoor logging w/o checking what it is running inside.
14470	*
14471	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14472	* backdoor port and magic numbers loaded in registers.
14473	*
14474	* @returns true if it is, false if it isn't.
14475	* @sa hmR0SvmIsMesaDrvGp.
14476	*/
14477	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14478	{
14479	/* 0xed: IN eAX,dx */
14480	uint8_t abInstr[1];
14481	if (pVmxTransient->cbInstr != sizeof(abInstr))
14482	return false;
14483
14484	/* Check that it is #GP(0). */
14485	if (pVmxTransient->uExitIntErrorCode != 0)
14486	return false;
14487
14488	/* Check magic and port. */
14489	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14490	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14491	if (pCtx->rax != UINT32_C(0x564d5868))
14492	return false;
14493	if (pCtx->dx != UINT32_C(0x5658))
14494	return false;
14495
14496	/* Flat ring-3 CS. */
14497	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14498	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14499	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14500	if (pCtx->cs.Attr.n.u2Dpl != 3)
14501	return false;
14502	if (pCtx->cs.u64Base != 0)
14503	return false;
14504
14505	/* Check opcode. */
14506	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14507	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14508	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14509	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14510	if (RT_FAILURE(rc))
14511	return false;
14512	if (abInstr[0] != 0xed)
14513	return false;
14514
14515	return true;
14516	}
14517
14518	/**
14519	* VM-exit exception handler for \#GP (General-protection exception).
14520	*
14521	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14522	*/
14523	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14524	{
14525	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14526	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14527
14528	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14529	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14530	if (pVmcsInfo->RealMode.fRealOnV86Active)
14531	{ /* likely */ }
14532	else
14533	{
14534	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14535	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14536	#endif
14537	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14538	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14539	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14540	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14541	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14542	AssertRCReturn(rc, rc);
14543	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14544	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14545
14546	if ( pVmxTransient->fIsNestedGuest
14547	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14548	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14549	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14550	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14551	else
14552	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14553	return rc;
14554	}
14555
14556	Assert(CPUMIsGuestInRealModeEx(pCtx));
14557	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14558	Assert(!pVmxTransient->fIsNestedGuest);
14559
14560	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14561	AssertRCReturn(rc, rc);
14562
14563	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14564	if (rcStrict == VINF_SUCCESS)
14565	{
14566	if (!CPUMIsGuestInRealModeEx(pCtx))
14567	{
14568	/*
14569	* The guest is no longer in real-mode, check if we can continue executing the
14570	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14571	*/
14572	pVmcsInfo->RealMode.fRealOnV86Active = false;
14573	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14574	{
14575	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14576	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14577	}
14578	else
14579	{
14580	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14581	rcStrict = VINF_EM_RESCHEDULE;
14582	}
14583	}
14584	else
14585	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14586	}
14587	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14588	{
14589	rcStrict = VINF_SUCCESS;
14590	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14591	}
14592	return VBOXSTRICTRC_VAL(rcStrict);
14593	}
14594
14595
14596	/**
14597	* VM-exit exception handler wrapper for generic exceptions.
14598	*
14599	* This simply re-injects the exception back into the VM without any special
14600	* processing.
14601	*/
14602	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14603	{
14604	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14605
14606	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14607	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14608	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14609	("uVector=%#x u32XcptBitmap=%#X32\n",
14610	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14611	NOREF(pVmcsInfo);
14612	#endif
14613
14614	/*
14615	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14616	* would have been handled while checking exits due to event delivery.
14617	*/
14618	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14619	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14620	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14621	AssertRCReturn(rc, rc);
14622
14623	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14624
14625	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14626	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14627	AssertRCReturn(rc, rc);
14628	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14629	#endif
14630
14631	#ifdef VBOX_WITH_STATISTICS
14632	switch (uVector)
14633	{
14634	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14635	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14636	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14637	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14638	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14639	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14640	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14641	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14642	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14643	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14644	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14645	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14646	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14647	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14648	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14649	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14650	default:
14651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14652	break;
14653	}
14654	#endif
14655
14656	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14657	Assert( uVector != X86_XCPT_PF
14658	\|\| VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo) != VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
14659	NOREF(uVector);
14660
14661	/* Re-inject the original exception into the guest. */
14662	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14663	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14664	return VINF_SUCCESS;
14665	}
14666	/** @} */
14667
14668
14669	/** @name VM-exit handlers.
14670	* @{
14671	*/
14672	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14673	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14674	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14675
14676	/**
14677	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14678	*/
14679	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14680	{
14681	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14682	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14683	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14684	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14685	return VINF_SUCCESS;
14686	return VINF_EM_RAW_INTERRUPT;
14687	}
14688
14689
14690	/**
14691	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14692	* VM-exit.
14693	*/
14694	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14695	{
14696	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14697	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14698
14699	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14700	AssertRCReturn(rc, rc);
14701
14702	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14703	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14704
14705	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14706	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14707	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14708	NOREF(pVmcsInfo);
14709
14710	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
14711	{
14712	/*
14713	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14714	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14715	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14716	*
14717	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14718	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
14719	*/
14720	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14721	return hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14722	}
14723
14724	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14725	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14726	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
14727	{ /* likely */ }
14728	else
14729	{
14730	if (rcStrict == VINF_HM_DOUBLE_FAULT)
14731	{
14732	Assert(pVCpu->hm.s.Event.fPending);
14733	rcStrict = VINF_SUCCESS;
14734	}
14735	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14736	return rcStrict;
14737	}
14738
14739	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
14740	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
14741	switch (uIntType)
14742	{
14743	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
14744	Assert(uVector == X86_XCPT_DB);
14745	RT_FALL_THRU();
14746	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
14747	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14748	RT_FALL_THRU();
14749	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14750	{
14751	/*
14752	* If there's any exception caused as a result of event injection, the resulting
14753	* secondary/final execption will be pending, we shall continue guest execution
14754	* after injecting the event. The page-fault case is complicated and we manually
14755	* handle any currently pending event in hmR0VmxExitXcptPF.
14756	*/
14757	if (!pVCpu->hm.s.Event.fPending)
14758	{ /* likely */ }
14759	else if (uVector != X86_XCPT_PF)
14760	{
14761	rcStrict = VINF_SUCCESS;
14762	break;
14763	}
14764
14765	switch (uVector)
14766	{
14767	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
14768	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
14769	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
14770	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
14771	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
14772	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
14773	default:
14774	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient);
14775	break;
14776	}
14777	break;
14778	}
14779
14780	default:
14781	{
14782	pVCpu->hm.s.u32HMError = uExitIntInfo;
14783	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14784	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
14785	break;
14786	}
14787	}
14788	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14789	return rcStrict;
14790	}
14791
14792
14793	/**
14794	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14795	*/
14796	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14797	{
14798	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14799
14800	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14801	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14802	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14803	AssertRCReturn(rc, rc);
14804
14805	/* Evaluate and deliver pending events and resume guest execution. */
14806	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14807	return VINF_SUCCESS;
14808	}
14809
14810
14811	/**
14812	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14813	*/
14814	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14815	{
14816	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14817
14818	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14819	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14820	{
14821	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14822	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14823	}
14824
14825	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14826
14827	/*
14828	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14829	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14830	*/
14831	uint32_t fIntrState;
14832	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14833	AssertRCReturn(rc, rc);
14834	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14835	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14836	{
14837	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14838	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14839
14840	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14841	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14842	AssertRCReturn(rc, rc);
14843	}
14844
14845	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14846	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14847	AssertRCReturn(rc, rc);
14848
14849	/* Evaluate and deliver pending events and resume guest execution. */
14850	return VINF_SUCCESS;
14851	}
14852
14853
14854	/**
14855	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14856	*/
14857	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14858	{
14859	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14860	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14861	}
14862
14863
14864	/**
14865	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14866	*/
14867	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14868	{
14869	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14870	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14871	}
14872
14873
14874	/**
14875	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14876	*/
14877	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14878	{
14879	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14880
14881	/*
14882	* Get the state we need and update the exit history entry.
14883	*/
14884	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14885	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14886	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14887	AssertRCReturn(rc, rc);
14888
14889	VBOXSTRICTRC rcStrict;
14890	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14891	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14892	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14893	if (!pExitRec)
14894	{
14895	/*
14896	* Regular CPUID instruction execution.
14897	*/
14898	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
14899	if (rcStrict == VINF_SUCCESS)
14900	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14901	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14902	{
14903	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14904	rcStrict = VINF_SUCCESS;
14905	}
14906	}
14907	else
14908	{
14909	/*
14910	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14911	*/
14912	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14913	AssertRCReturn(rc2, rc2);
14914
14915	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14916	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14917
14918	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14919	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14920
14921	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14922	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14923	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14924	}
14925	return rcStrict;
14926	}
14927
14928
14929	/**
14930	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14931	*/
14932	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14933	{
14934	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14935
14936	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14937	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14938	AssertRCReturn(rc, rc);
14939
14940	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14941	return VINF_EM_RAW_EMULATE_INSTR;
14942
14943	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14944	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14945	}
14946
14947
14948	/**
14949	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14950	*/
14951	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14952	{
14953	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14954
14955	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14956	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14957	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14958	AssertRCReturn(rc, rc);
14959
14960	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
14961	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14962	{
14963	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14964	we must reset offsetting on VM-entry. See @bugref{6634}. */
14965	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14966	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14967	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14968	}
14969	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14970	{
14971	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14972	rcStrict = VINF_SUCCESS;
14973	}
14974	return rcStrict;
14975	}
14976
14977
14978	/**
14979	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14980	*/
14981	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14982	{
14983	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14984
14985	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14986	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14987	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14988	AssertRCReturn(rc, rc);
14989
14990	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
14991	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14992	{
14993	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14994	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14995	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14996	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14997	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14998	}
14999	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15000	{
15001	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15002	rcStrict = VINF_SUCCESS;
15003	}
15004	return rcStrict;
15005	}
15006
15007
15008	/**
15009	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
15010	*/
15011	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15012	{
15013	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15014
15015	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15016	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
15017	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
15018	AssertRCReturn(rc, rc);
15019
15020	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15021	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15022	if (RT_LIKELY(rc == VINF_SUCCESS))
15023	{
15024	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15025	Assert(pVmxTransient->cbInstr == 2);
15026	}
15027	else
15028	{
15029	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
15030	rc = VERR_EM_INTERPRETER;
15031	}
15032	return rc;
15033	}
15034
15035
15036	/**
15037	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
15038	*/
15039	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15040	{
15041	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15042
15043	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
15044	if (EMAreHypercallInstructionsEnabled(pVCpu))
15045	{
15046	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15047	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
15048	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
15049	AssertRCReturn(rc, rc);
15050
15051	/* Perform the hypercall. */
15052	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
15053	if (rcStrict == VINF_SUCCESS)
15054	{
15055	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15056	AssertRCReturn(rc, rc);
15057	}
15058	else
15059	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
15060	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
15061	\|\| RT_FAILURE(rcStrict));
15062
15063	/* If the hypercall changes anything other than guest's general-purpose registers,
15064	we would need to reload the guest changed bits here before VM-entry. */
15065	}
15066	else
15067	Log4Func(("Hypercalls not enabled\n"));
15068
15069	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
15070	if (RT_FAILURE(rcStrict))
15071	{
15072	hmR0VmxSetPendingXcptUD(pVCpu);
15073	rcStrict = VINF_SUCCESS;
15074	}
15075
15076	return rcStrict;
15077	}
15078
15079
15080	/**
15081	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
15082	*/
15083	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15084	{
15085	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15086	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
15087
15088	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15089	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15090	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15091	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15092	AssertRCReturn(rc, rc);
15093
15094	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
15095
15096	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
15097	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15098	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15099	{
15100	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15101	rcStrict = VINF_SUCCESS;
15102	}
15103	else
15104	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
15105	VBOXSTRICTRC_VAL(rcStrict)));
15106	return rcStrict;
15107	}
15108
15109
15110	/**
15111	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
15112	*/
15113	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15114	{
15115	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15116
15117	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15118	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
15119	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15120	AssertRCReturn(rc, rc);
15121
15122	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
15123	if (rcStrict == VINF_SUCCESS)
15124	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15125	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15126	{
15127	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15128	rcStrict = VINF_SUCCESS;
15129	}
15130
15131	return rcStrict;
15132	}
15133
15134
15135	/**
15136	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
15137	*/
15138	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15139	{
15140	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15141
15142	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15143	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15144	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15145	AssertRCReturn(rc, rc);
15146
15147	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
15148	if (RT_SUCCESS(rcStrict))
15149	{
15150	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15151	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
15152	rcStrict = VINF_SUCCESS;
15153	}
15154
15155	return rcStrict;
15156	}
15157
15158
15159	/**
15160	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
15161	* VM-exit.
15162	*/
15163	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15164	{
15165	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15166	return VINF_EM_RESET;
15167	}
15168
15169
15170	/**
15171	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
15172	*/
15173	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15174	{
15175	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15176
15177	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15178	AssertRCReturn(rc, rc);
15179
15180	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
15181	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
15182	rc = VINF_SUCCESS;
15183	else
15184	rc = VINF_EM_HALT;
15185
15186	if (rc != VINF_SUCCESS)
15187	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
15188	return rc;
15189	}
15190
15191
15192	/**
15193	* VM-exit handler for instructions that result in a \#UD exception delivered to
15194	* the guest.
15195	*/
15196	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15197	{
15198	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15199	hmR0VmxSetPendingXcptUD(pVCpu);
15200	return VINF_SUCCESS;
15201	}
15202
15203
15204	/**
15205	* VM-exit handler for expiry of the VMX-preemption timer.
15206	*/
15207	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15208	{
15209	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15210
15211	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
15212	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15213
15214	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
15215	PVM pVM = pVCpu->CTX_SUFF(pVM);
15216	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
15217	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
15218	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
15219	}
15220
15221
15222	/**
15223	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
15224	*/
15225	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15226	{
15227	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15228
15229	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15230	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15231	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
15232	AssertRCReturn(rc, rc);
15233
15234	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
15235	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15236	: HM_CHANGED_RAISED_XCPT_MASK);
15237
15238	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15239	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
15240
15241	return rcStrict;
15242	}
15243
15244
15245	/**
15246	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
15247	*/
15248	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15249	{
15250	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15251	/** @todo Use VM-exit instruction information. */
15252	return VERR_EM_INTERPRETER;
15253	}
15254
15255
15256	/**
15257	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
15258	* VM-exit.
15259	*/
15260	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15261	{
15262	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15263	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15264	AssertRCReturn(rc, rc);
15265
15266	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
15267	if (RT_FAILURE(rc))
15268	return rc;
15269
15270	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
15271	NOREF(uInvalidReason);
15272
15273	#ifdef VBOX_STRICT
15274	uint32_t fIntrState;
15275	RTHCUINTREG uHCReg;
15276	uint64_t u64Val;
15277	uint32_t u32Val;
15278	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
15279	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
15280	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
15281	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
15282	AssertRCReturn(rc, rc);
15283
15284	Log4(("uInvalidReason %u\n", uInvalidReason));
15285	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
15286	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
15287	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
15288	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
15289
15290	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
15291	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
15292	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
15293	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
15294	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
15295	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15296	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
15297	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
15298	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
15299	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15300	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
15301	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
15302
15303	hmR0DumpRegs(pVCpu);
15304	#endif
15305
15306	return VERR_VMX_INVALID_GUEST_STATE;
15307	}
15308
15309	/**
15310	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
15311	*/
15312	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15313	{
15314	/*
15315	* Cummulative notes of all recognized but unexpected VM-exits.
15316	*
15317	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
15318	* nested-paging is used.
15319	*
15320	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
15321	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
15322	* this function (and thereby stop VM execution) for handling such instructions.
15323	*
15324	*
15325	* VMX_EXIT_INIT_SIGNAL:
15326	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
15327	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
15328	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
15329	*
15330	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
15331	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
15332	* See Intel spec. "23.8 Restrictions on VMX operation".
15333	*
15334	* VMX_EXIT_SIPI:
15335	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
15336	* activity state is used. We don't make use of it as our guests don't have direct
15337	* access to the host local APIC.
15338	*
15339	* See Intel spec. 25.3 "Other Causes of VM-exits".
15340	*
15341	* VMX_EXIT_IO_SMI:
15342	* VMX_EXIT_SMI:
15343	* This can only happen if we support dual-monitor treatment of SMI, which can be
15344	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
15345	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
15346	* VMX root mode or receive an SMI. If we get here, something funny is going on.
15347	*
15348	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
15349	* See Intel spec. 25.3 "Other Causes of VM-Exits"
15350	*
15351	* VMX_EXIT_ERR_MSR_LOAD:
15352	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
15353	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
15354	* execution.
15355	*
15356	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
15357	*
15358	* VMX_EXIT_ERR_MACHINE_CHECK:
15359	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
15360	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
15361	* #MC exception abort class exception is raised. We thus cannot assume a
15362	* reasonable chance of continuing any sort of execution and we bail.
15363	*
15364	* See Intel spec. 15.1 "Machine-check Architecture".
15365	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
15366	*
15367	* VMX_EXIT_PML_FULL:
15368	* VMX_EXIT_VIRTUALIZED_EOI:
15369	* VMX_EXIT_APIC_WRITE:
15370	* We do not currently support any of these features and thus they are all unexpected
15371	* VM-exits.
15372	*
15373	* VMX_EXIT_GDTR_IDTR_ACCESS:
15374	* VMX_EXIT_LDTR_TR_ACCESS:
15375	* VMX_EXIT_RDRAND:
15376	* VMX_EXIT_RSM:
15377	* VMX_EXIT_VMFUNC:
15378	* VMX_EXIT_ENCLS:
15379	* VMX_EXIT_RDSEED:
15380	* VMX_EXIT_XSAVES:
15381	* VMX_EXIT_XRSTORS:
15382	* VMX_EXIT_UMWAIT:
15383	* VMX_EXIT_TPAUSE:
15384	* These VM-exits are -not- caused unconditionally by execution of the corresponding
15385	* instruction. Any VM-exit for these instructions indicate a hardware problem,
15386	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
15387	*
15388	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
15389	*/
15390	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15391	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
15392	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15393	}
15394
15395
15396	/**
15397	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
15398	*/
15399	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15400	{
15401	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15402
15403	/** @todo Optimize this: We currently drag in in the whole MSR state
15404	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15405	* MSRs required. That would require changes to IEM and possibly CPUM too.
15406	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15407	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15408	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15409	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15410	switch (idMsr)
15411	{
15412	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15413	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15414	}
15415
15416	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15417	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15418	AssertRCReturn(rc, rc);
15419
15420	Log4Func(("ecx=%#RX32\n", idMsr));
15421
15422	#ifdef VBOX_STRICT
15423	Assert(!pVmxTransient->fIsNestedGuest);
15424	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15425	{
15426	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15427	&& idMsr != MSR_K6_EFER)
15428	{
15429	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15430	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15431	}
15432	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15433	{
15434	Assert(pVmcsInfo->pvMsrBitmap);
15435	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15436	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15437	{
15438	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15439	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15440	}
15441	}
15442	}
15443	#endif
15444
15445	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
15446	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15447	if (rcStrict == VINF_SUCCESS)
15448	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15449	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
15450	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15451	{
15452	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15453	rcStrict = VINF_SUCCESS;
15454	}
15455	else
15456	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15457
15458	return rcStrict;
15459	}
15460
15461
15462	/**
15463	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15464	*/
15465	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15466	{
15467	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15468
15469	/** @todo Optimize this: We currently drag in in the whole MSR state
15470	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15471	* MSRs required. That would require changes to IEM and possibly CPUM too.
15472	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15473	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15474	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15475
15476	/*
15477	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15478	* Although we don't need to fetch the base as it will be overwritten shortly, while
15479	* loading guest-state we would also load the entire segment register including limit
15480	* and attributes and thus we need to load them here.
15481	*/
15482	switch (idMsr)
15483	{
15484	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15485	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15486	}
15487
15488	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15489	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15490	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15491	AssertRCReturn(rc, rc);
15492
15493	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15494
15495	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
15496	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15497
15498	if (rcStrict == VINF_SUCCESS)
15499	{
15500	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15501
15502	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15503	if ( idMsr == MSR_IA32_APICBASE
15504	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15505	&& idMsr <= MSR_IA32_X2APIC_END))
15506	{
15507	/*
15508	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15509	* When full APIC register virtualization is implemented we'll have to make
15510	* sure APIC state is saved from the VMCS before IEM changes it.
15511	*/
15512	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15513	}
15514	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15515	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15516	else if (idMsr == MSR_K6_EFER)
15517	{
15518	/*
15519	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15520	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15521	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15522	*/
15523	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15524	}
15525
15526	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15527	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15528	{
15529	switch (idMsr)
15530	{
15531	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15532	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15533	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15534	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15535	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15536	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15537	default:
15538	{
15539	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15540	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15541	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15542	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15543	break;
15544	}
15545	}
15546	}
15547	#ifdef VBOX_STRICT
15548	else
15549	{
15550	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15551	switch (idMsr)
15552	{
15553	case MSR_IA32_SYSENTER_CS:
15554	case MSR_IA32_SYSENTER_EIP:
15555	case MSR_IA32_SYSENTER_ESP:
15556	case MSR_K8_FS_BASE:
15557	case MSR_K8_GS_BASE:
15558	{
15559	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15560	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15561	}
15562
15563	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15564	default:
15565	{
15566	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15567	{
15568	/* EFER MSR writes are always intercepted. */
15569	if (idMsr != MSR_K6_EFER)
15570	{
15571	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15572	idMsr));
15573	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15574	}
15575	}
15576
15577	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15578	{
15579	Assert(pVmcsInfo->pvMsrBitmap);
15580	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15581	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15582	{
15583	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15584	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15585	}
15586	}
15587	break;
15588	}
15589	}
15590	}
15591	#endif /* VBOX_STRICT */
15592	}
15593	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15594	{
15595	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15596	rcStrict = VINF_SUCCESS;
15597	}
15598	else
15599	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15600
15601	return rcStrict;
15602	}
15603
15604
15605	/**
15606	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15607	*/
15608	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15609	{
15610	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15611
15612	/** @todo The guest has likely hit a contended spinlock. We might want to
15613	* poke a schedule different guest VCPU. */
15614	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15615	if (RT_SUCCESS(rc))
15616	return VINF_EM_RAW_INTERRUPT;
15617
15618	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15619	return rc;
15620	}
15621
15622
15623	/**
15624	* VM-exit handler for when the TPR value is lowered below the specified
15625	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15626	*/
15627	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15628	{
15629	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15630	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15631
15632	/*
15633	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15634	* We'll re-evaluate pending interrupts and inject them before the next VM
15635	* entry so we can just continue execution here.
15636	*/
15637	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15638	return VINF_SUCCESS;
15639	}
15640
15641
15642	/**
15643	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15644	* VM-exit.
15645	*
15646	* @retval VINF_SUCCESS when guest execution can continue.
15647	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15648	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15649	* incompatible guest state for VMX execution (real-on-v86 case).
15650	*/
15651	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15652	{
15653	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15654	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15655
15656	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15657	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15658	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15659	AssertRCReturn(rc, rc);
15660
15661	VBOXSTRICTRC rcStrict;
15662	PVM pVM = pVCpu->CTX_SUFF(pVM);
15663	uint64_t const uExitQual = pVmxTransient->uExitQual;
15664	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15665	switch (uAccessType)
15666	{
15667	/*
15668	* MOV to CRx.
15669	*/
15670	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15671	{
15672	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15673	AssertRCReturn(rc, rc);
15674
15675	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15676	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15677	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15678	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15679
15680	/*
15681	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15682	* - When nested paging isn't used.
15683	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15684	* - We are executing in the VM debug loop.
15685	*/
15686	Assert( iCrReg != 3
15687	\|\| !pVM->hm.s.fNestedPaging
15688	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15689	\|\| pVCpu->hm.s.fUsingDebugLoop);
15690
15691	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15692	Assert( iCrReg != 8
15693	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15694
15695	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15696	AssertMsg( rcStrict == VINF_SUCCESS
15697	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15698
15699	/*
15700	* This is a kludge for handling switches back to real mode when we try to use
15701	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15702	* deal with special selector values, so we have to return to ring-3 and run
15703	* there till the selector values are V86 mode compatible.
15704	*
15705	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15706	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15707	* this function.
15708	*/
15709	if ( iCrReg == 0
15710	&& rcStrict == VINF_SUCCESS
15711	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15712	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15713	&& (uOldCr0 & X86_CR0_PE)
15714	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15715	{
15716	/** @todo Check selectors rather than returning all the time. */
15717	Assert(!pVmxTransient->fIsNestedGuest);
15718	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15719	rcStrict = VINF_EM_RESCHEDULE_REM;
15720	}
15721	break;
15722	}
15723
15724	/*
15725	* MOV from CRx.
15726	*/
15727	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15728	{
15729	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15730	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15731
15732	/*
15733	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15734	* - When nested paging isn't used.
15735	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15736	* - We are executing in the VM debug loop.
15737	*/
15738	Assert( iCrReg != 3
15739	\|\| !pVM->hm.s.fNestedPaging
15740	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15741	\|\| pVCpu->hm.s.fUsingDebugLoop);
15742
15743	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15744	Assert( iCrReg != 8
15745	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15746
15747	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15748	break;
15749	}
15750
15751	/*
15752	* CLTS (Clear Task-Switch Flag in CR0).
15753	*/
15754	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15755	{
15756	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15757	break;
15758	}
15759
15760	/*
15761	* LMSW (Load Machine-Status Word into CR0).
15762	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15763	*/
15764	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15765	{
15766	RTGCPTR GCPtrEffDst;
15767	uint8_t const cbInstr = pVmxTransient->cbInstr;
15768	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15769	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15770	if (fMemOperand)
15771	{
15772	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
15773	AssertRCReturn(rc, rc);
15774	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15775	}
15776	else
15777	GCPtrEffDst = NIL_RTGCPTR;
15778	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15779	break;
15780	}
15781
15782	default:
15783	{
15784	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15785	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15786	}
15787	}
15788
15789	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15790	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15791	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15792
15793	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15794	NOREF(pVM);
15795	return rcStrict;
15796	}
15797
15798
15799	/**
15800	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15801	* VM-exit.
15802	*/
15803	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15804	{
15805	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15806	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15807
15808	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15809	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15810	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15811	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15812	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15813	\| CPUMCTX_EXTRN_EFER);
15814	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15815	AssertRCReturn(rc, rc);
15816
15817	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15818	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15819	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15820	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15821	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15822	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15823	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15824	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15825
15826	/*
15827	* Update exit history to see if this exit can be optimized.
15828	*/
15829	VBOXSTRICTRC rcStrict;
15830	PCEMEXITREC pExitRec = NULL;
15831	if ( !fGstStepping
15832	&& !fDbgStepping)
15833	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15834	!fIOString
15835	? !fIOWrite
15836	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15837	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15838	: !fIOWrite
15839	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15840	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15841	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15842	if (!pExitRec)
15843	{
15844	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15845	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15846
15847	uint32_t const cbValue = s_aIOSizes[uIOSize];
15848	uint32_t const cbInstr = pVmxTransient->cbInstr;
15849	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15850	PVM pVM = pVCpu->CTX_SUFF(pVM);
15851	if (fIOString)
15852	{
15853	/*
15854	* INS/OUTS - I/O String instruction.
15855	*
15856	* Use instruction-information if available, otherwise fall back on
15857	* interpreting the instruction.
15858	*/
15859	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15860	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15861	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15862	if (fInsOutsInfo)
15863	{
15864	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15865	AssertRCReturn(rc2, rc2);
15866	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15867	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15868	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15869	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15870	if (fIOWrite)
15871	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15872	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15873	else
15874	{
15875	/*
15876	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15877	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15878	* See Intel Instruction spec. for "INS".
15879	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15880	*/
15881	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15882	}
15883	}
15884	else
15885	rcStrict = IEMExecOne(pVCpu);
15886
15887	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15888	fUpdateRipAlready = true;
15889	}
15890	else
15891	{
15892	/*
15893	* IN/OUT - I/O instruction.
15894	*/
15895	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15896	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15897	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15898	if (fIOWrite)
15899	{
15900	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15901	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15902	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15903	&& !pCtx->eflags.Bits.u1TF)
15904	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15905	}
15906	else
15907	{
15908	uint32_t u32Result = 0;
15909	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15910	if (IOM_SUCCESS(rcStrict))
15911	{
15912	/* Save result of I/O IN instr. in AL/AX/EAX. */
15913	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15914	}
15915	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15916	&& !pCtx->eflags.Bits.u1TF)
15917	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15918	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15919	}
15920	}
15921
15922	if (IOM_SUCCESS(rcStrict))
15923	{
15924	if (!fUpdateRipAlready)
15925	{
15926	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15927	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15928	}
15929
15930	/*
15931	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15932	* while booting Fedora 17 64-bit guest.
15933	*
15934	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15935	*/
15936	if (fIOString)
15937	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15938
15939	/*
15940	* If any I/O breakpoints are armed, we need to check if one triggered
15941	* and take appropriate action.
15942	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15943	*/
15944	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15945	AssertRCReturn(rc, rc);
15946
15947	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15948	* execution engines about whether hyper BPs and such are pending. */
15949	uint32_t const uDr7 = pCtx->dr[7];
15950	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15951	&& X86_DR7_ANY_RW_IO(uDr7)
15952	&& (pCtx->cr4 & X86_CR4_DE))
15953	\|\| DBGFBpIsHwIoArmed(pVM)))
15954	{
15955	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15956
15957	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15958	VMMRZCallRing3Disable(pVCpu);
15959	HM_DISABLE_PREEMPT(pVCpu);
15960
15961	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15962
15963	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15964	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15965	{
15966	/* Raise #DB. */
15967	if (fIsGuestDbgActive)
15968	ASMSetDR6(pCtx->dr[6]);
15969	if (pCtx->dr[7] != uDr7)
15970	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15971
15972	hmR0VmxSetPendingXcptDB(pVCpu);
15973	}
15974	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15975	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15976	else if ( rcStrict2 != VINF_SUCCESS
15977	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15978	rcStrict = rcStrict2;
15979	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15980
15981	HM_RESTORE_PREEMPT();
15982	VMMRZCallRing3Enable(pVCpu);
15983	}
15984	}
15985
15986	#ifdef VBOX_STRICT
15987	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15988	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15989	Assert(!fIOWrite);
15990	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15991	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15992	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15993	Assert(fIOWrite);
15994	else
15995	{
15996	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15997	* statuses, that the VMM device and some others may return. See
15998	* IOM_SUCCESS() for guidance. */
15999	AssertMsg( RT_FAILURE(rcStrict)
16000	\|\| rcStrict == VINF_SUCCESS
16001	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
16002	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
16003	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
16004	\|\| rcStrict == VINF_EM_RAW_TO_R3
16005	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16006	# endif
16007	}
16008	#endif
16009	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
16010	}
16011	else
16012	{
16013	/*
16014	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
16015	*/
16016	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16017	AssertRCReturn(rc2, rc2);
16018	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
16019	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
16020	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
16021	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16022	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
16023	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
16024
16025	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16026	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16027
16028	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16029	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16030	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16031	}
16032	return rcStrict;
16033	}
16034
16035
16036	/**
16037	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
16038	* VM-exit.
16039	*/
16040	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16041	{
16042	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16043
16044	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
16045	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16046	AssertRCReturn(rc, rc);
16047	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
16048	{
16049	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16050	AssertRCReturn(rc, rc);
16051	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16052	{
16053	uint32_t uErrCode;
16054	RTGCUINTPTR GCPtrFaultAddress;
16055	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
16056	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
16057	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
16058	if (fErrorCodeValid)
16059	{
16060	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16061	AssertRCReturn(rc, rc);
16062	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
16063	}
16064	else
16065	uErrCode = 0;
16066
16067	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
16068	&& uVector == X86_XCPT_PF)
16069	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
16070	else
16071	GCPtrFaultAddress = 0;
16072
16073	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16074	AssertRCReturn(rc, rc);
16075
16076	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
16077	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
16078
16079	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
16080	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16081	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16082	}
16083	}
16084
16085	/* Fall back to the interpreter to emulate the task-switch. */
16086	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16087	return VERR_EM_INTERPRETER;
16088	}
16089
16090
16091	/**
16092	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
16093	*/
16094	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16095	{
16096	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16097
16098	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16099	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
16100	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16101	AssertRCReturn(rc, rc);
16102	return VINF_EM_DBG_STEPPED;
16103	}
16104
16105
16106	/**
16107	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
16108	*/
16109	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16110	{
16111	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16112	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
16113
16114	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16115	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16116	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16117	{
16118	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
16119	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16120	{
16121	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
16122	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16123	}
16124	}
16125	else
16126	{
16127	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16128	{
16129	Assert(pVCpu->hm.s.Event.fPending);
16130	rcStrict1 = VINF_SUCCESS;
16131	}
16132	return rcStrict1;
16133	}
16134
16135	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
16136	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16137	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16138	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16139	AssertRCReturn(rc, rc);
16140
16141	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
16142	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
16143	VBOXSTRICTRC rcStrict2;
16144	switch (uAccessType)
16145	{
16146	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
16147	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
16148	{
16149	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
16150	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
16151	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
16152
16153	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
16154	GCPhys &= PAGE_BASE_GC_MASK;
16155	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
16156	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
16157	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
16158
16159	PVM pVM = pVCpu->CTX_SUFF(pVM);
16160	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16161	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
16162	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
16163	CPUMCTX2CORE(pCtx), GCPhys);
16164	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
16165	if ( rcStrict2 == VINF_SUCCESS
16166	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
16167	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
16168	{
16169	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16170	\| HM_CHANGED_GUEST_APIC_TPR);
16171	rcStrict2 = VINF_SUCCESS;
16172	}
16173	break;
16174	}
16175
16176	default:
16177	{
16178	Log4Func(("uAccessType=%#x\n", uAccessType));
16179	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
16180	break;
16181	}
16182	}
16183
16184	if (rcStrict2 != VINF_SUCCESS)
16185	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
16186	return rcStrict2;
16187	}
16188
16189
16190	/**
16191	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
16192	* VM-exit.
16193	*/
16194	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16195	{
16196	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16197
16198	/* We should -not- get this VM-exit if the guest's debug registers were active. */
16199	if (pVmxTransient->fWasGuestDebugStateActive)
16200	{
16201	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
16202	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
16203	}
16204
16205	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16206	if ( !pVCpu->hm.s.fSingleInstruction
16207	&& !pVmxTransient->fWasHyperDebugStateActive)
16208	{
16209	Assert(!DBGFIsStepping(pVCpu));
16210	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
16211
16212	/* Don't intercept MOV DRx any more. */
16213	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
16214	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16215	AssertRCReturn(rc, rc);
16216
16217	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
16218	VMMRZCallRing3Disable(pVCpu);
16219	HM_DISABLE_PREEMPT(pVCpu);
16220
16221	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
16222	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
16223	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
16224
16225	HM_RESTORE_PREEMPT();
16226	VMMRZCallRing3Enable(pVCpu);
16227
16228	#ifdef VBOX_WITH_STATISTICS
16229	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16230	AssertRCReturn(rc, rc);
16231	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16232	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16233	else
16234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16235	#endif
16236	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
16237	return VINF_SUCCESS;
16238	}
16239
16240	/*
16241	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
16242	* The EFER MSR is always up-to-date.
16243	* Update the segment registers and DR7 from the CPU.
16244	*/
16245	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16246	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16247	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
16248	AssertRCReturn(rc, rc);
16249	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
16250
16251	PVM pVM = pVCpu->CTX_SUFF(pVM);
16252	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16253	{
16254	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16255	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
16256	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
16257	if (RT_SUCCESS(rc))
16258	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
16259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16260	}
16261	else
16262	{
16263	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16264	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
16265	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
16266	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16267	}
16268
16269	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
16270	if (RT_SUCCESS(rc))
16271	{
16272	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
16273	AssertRCReturn(rc2, rc2);
16274	return VINF_SUCCESS;
16275	}
16276	return rc;
16277	}
16278
16279
16280	/**
16281	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
16282	* Conditional VM-exit.
16283	*/
16284	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16285	{
16286	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16287	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16288
16289	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16290	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16291	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16292	{
16293	/*
16294	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
16295	* instruction emulation to inject the original event. Otherwise, injecting the original event
16296	* using hardware-assisted VMX would would trigger the same EPT misconfig VM-exit again.
16297	*/
16298	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16299	{
16300	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
16301	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16302	}
16303	}
16304	else
16305	{
16306	if (rcStrict == VINF_HM_DOUBLE_FAULT)
16307	rcStrict = VINF_SUCCESS;
16308	return rcStrict;
16309	}
16310
16311	/*
16312	* Get sufficent state and update the exit history entry.
16313	*/
16314	RTGCPHYS GCPhys;
16315	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16316	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16317	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16318	AssertRCReturn(rc, rc);
16319
16320	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
16321	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
16322	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
16323	if (!pExitRec)
16324	{
16325	/*
16326	* If we succeed, resume guest execution.
16327	* If we fail in interpreting the instruction because we couldn't get the guest physical address
16328	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
16329	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
16330	* weird case. See @bugref{6043}.
16331	*/
16332	PVM pVM = pVCpu->CTX_SUFF(pVM);
16333	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16334	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
16335	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
16336	if ( rcStrict == VINF_SUCCESS
16337	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16338	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16339	{
16340	/* Successfully handled MMIO operation. */
16341	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16342	\| HM_CHANGED_GUEST_APIC_TPR);
16343	rcStrict = VINF_SUCCESS;
16344	}
16345	}
16346	else
16347	{
16348	/*
16349	* Frequent exit or something needing probing. Call EMHistoryExec.
16350	*/
16351	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
16352	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
16353
16354	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16355	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16356
16357	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16358	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16359	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16360	}
16361	return rcStrict;
16362	}
16363
16364
16365	/**
16366	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
16367	* VM-exit.
16368	*/
16369	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16370	{
16371	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16372	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16373
16374	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16375	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16376	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16377	{
16378	/*
16379	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
16380	* we shall resolve the nested #PF and re-inject the original event.
16381	*/
16382	if (pVCpu->hm.s.Event.fPending)
16383	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
16384	}
16385	else
16386	{
16387	if (rcStrict == VINF_HM_DOUBLE_FAULT)
16388	{
16389	Assert(pVCpu->hm.s.Event.fPending);
16390	rcStrict = VINF_SUCCESS;
16391	}
16392	return rcStrict;
16393	}
16394
16395	RTGCPHYS GCPhys;
16396	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16397	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16398	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16399	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16400	AssertRCReturn(rc, rc);
16401
16402	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
16403	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
16404
16405	RTGCUINT uErrorCode = 0;
16406	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16407	uErrorCode \|= X86_TRAP_PF_ID;
16408	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16409	uErrorCode \|= X86_TRAP_PF_RW;
16410	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16411	uErrorCode \|= X86_TRAP_PF_P;
16412
16413	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16414
16415
16416	/* Handle the pagefault trap for the nested shadow table. */
16417	PVM pVM = pVCpu->CTX_SUFF(pVM);
16418	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16419
16420	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
16421	pCtx->cs.Sel, pCtx->rip));
16422
16423	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16424	TRPMResetTrap(pVCpu);
16425
16426	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16427	if ( rcStrict == VINF_SUCCESS
16428	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16429	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16430	{
16431	/* Successfully synced our nested page tables. */
16432	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16433	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16434	return VINF_SUCCESS;
16435	}
16436
16437	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16438	return rcStrict;
16439	}
16440
16441
16442	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16443	/**
16444	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16445	*/
16446	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16447	{
16448	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16449
16450	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16451	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16452	\| CPUMCTX_EXTRN_HWVIRT
16453	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16454	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16455	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16456	AssertRCReturn(rc, rc);
16457
16458	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16459
16460	VMXVEXITINFO ExitInfo;
16461	RT_ZERO(ExitInfo);
16462	ExitInfo.uReason = pVmxTransient->uExitReason;
16463	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16464	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16465	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16466	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16467
16468	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16469	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16470	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16471	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16472	{
16473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16474	rcStrict = VINF_SUCCESS;
16475	}
16476	return rcStrict;
16477	}
16478
16479
16480	/**
16481	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16482	*/
16483	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16484	{
16485	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16486
16487	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16488	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16489	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16490	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16491	AssertRCReturn(rc, rc);
16492
16493	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16494
16495	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16496	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16497	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16498	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16499	{
16500	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16501	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16502	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16503	}
16504	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16505	return rcStrict;
16506	}
16507
16508
16509	/**
16510	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16511	*/
16512	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16513	{
16514	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16515
16516	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16517	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16518	\| CPUMCTX_EXTRN_HWVIRT
16519	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16520	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16521	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16522	AssertRCReturn(rc, rc);
16523
16524	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16525
16526	VMXVEXITINFO ExitInfo;
16527	RT_ZERO(ExitInfo);
16528	ExitInfo.uReason = pVmxTransient->uExitReason;
16529	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16530	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16531	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16532	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16533
16534	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16535	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16536	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16537	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16538	{
16539	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16540	rcStrict = VINF_SUCCESS;
16541	}
16542	return rcStrict;
16543	}
16544
16545
16546	/**
16547	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16548	*/
16549	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16550	{
16551	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16552
16553	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16554	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16555	\| CPUMCTX_EXTRN_HWVIRT
16556	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16557	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16558	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16559	AssertRCReturn(rc, rc);
16560
16561	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16562
16563	VMXVEXITINFO ExitInfo;
16564	RT_ZERO(ExitInfo);
16565	ExitInfo.uReason = pVmxTransient->uExitReason;
16566	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16567	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16568	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16569	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16570
16571	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16572	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16573	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16574	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16575	{
16576	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16577	rcStrict = VINF_SUCCESS;
16578	}
16579	return rcStrict;
16580	}
16581
16582
16583	/**
16584	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16585	*/
16586	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16587	{
16588	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16589
16590	/*
16591	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16592	* thus might not need to import the shadow VMCS state, it's safer just in case
16593	* code elsewhere dares look at unsynced VMCS fields.
16594	*/
16595	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16596	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16597	\| CPUMCTX_EXTRN_HWVIRT
16598	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16599	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16600	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16601	AssertRCReturn(rc, rc);
16602
16603	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16604
16605	VMXVEXITINFO ExitInfo;
16606	RT_ZERO(ExitInfo);
16607	ExitInfo.uReason = pVmxTransient->uExitReason;
16608	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16609	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16610	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16611	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16612	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16613
16614	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16615	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16616	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16617	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16618	{
16619	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16620	rcStrict = VINF_SUCCESS;
16621	}
16622	return rcStrict;
16623	}
16624
16625
16626	/**
16627	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16628	*/
16629	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16630	{
16631	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16632
16633	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16634	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16635	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16636	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16637	AssertRCReturn(rc, rc);
16638
16639	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16640
16641	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16642	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16643	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16644	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16645	{
16646	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16647	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16648	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16649	}
16650	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16651	return rcStrict;
16652	}
16653
16654
16655	/**
16656	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16657	*/
16658	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16659	{
16660	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16661
16662	/*
16663	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16664	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16665	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16666	*/
16667	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16668	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16669	\| CPUMCTX_EXTRN_HWVIRT
16670	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16671	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16672	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16673	AssertRCReturn(rc, rc);
16674
16675	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16676
16677	VMXVEXITINFO ExitInfo;
16678	RT_ZERO(ExitInfo);
16679	ExitInfo.uReason = pVmxTransient->uExitReason;
16680	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16681	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16682	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16683	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16684	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16685
16686	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16687	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16688	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16689	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16690	{
16691	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16692	rcStrict = VINF_SUCCESS;
16693	}
16694	return rcStrict;
16695	}
16696
16697
16698	/**
16699	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16700	*/
16701	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16702	{
16703	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16704
16705	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16706	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16707	\| CPUMCTX_EXTRN_HWVIRT
16708	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16709	AssertRCReturn(rc, rc);
16710
16711	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16712
16713	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16714	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16715	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16716	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16717	{
16718	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16719	rcStrict = VINF_SUCCESS;
16720	}
16721	return rcStrict;
16722	}
16723
16724
16725	/**
16726	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16727	*/
16728	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16729	{
16730	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16731
16732	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16733	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16734	\| CPUMCTX_EXTRN_HWVIRT
16735	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16736	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16737	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16738	AssertRCReturn(rc, rc);
16739
16740	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16741
16742	VMXVEXITINFO ExitInfo;
16743	RT_ZERO(ExitInfo);
16744	ExitInfo.uReason = pVmxTransient->uExitReason;
16745	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16746	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16747	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16748	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16749
16750	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16751	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16752	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16753	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16754	{
16755	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16756	rcStrict = VINF_SUCCESS;
16757	}
16758	return rcStrict;
16759	}
16760
16761
16762	/**
16763	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16764	*/
16765	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16766	{
16767	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16768
16769	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16770	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16771	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16772	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16773	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16774	AssertRCReturn(rc, rc);
16775
16776	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16777
16778	VMXVEXITINFO ExitInfo;
16779	RT_ZERO(ExitInfo);
16780	ExitInfo.uReason = pVmxTransient->uExitReason;
16781	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16782	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16783	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16784	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16785
16786	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16787	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16788	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16789	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16790	{
16791	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16792	rcStrict = VINF_SUCCESS;
16793	}
16794	return rcStrict;
16795	}
16796	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16797	/** @} */
16798
16799
16800	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16801	/** @name Nested-guest VM-exit handlers.
16802	* @{
16803	*/
16804	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16805	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16806	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16807
16808	/**
16809	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16810	* Conditional VM-exit.
16811	*/
16812	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16813	{
16814	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16815
16816	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16817	AssertRCReturn(rc, rc);
16818
16819	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16820	uint32_t const uExtIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16821	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16822
16823	/*
16824	* Make sure not to use stale/previous VM-exit instruction length since we read the
16825	* instruction length from the VMCS below only for software exceptions and privileged
16826	* software exceptions but we pass it for all exception VM-exits below.
16827	*/
16828	pVmxTransient->cbInstr = 0;
16829
16830	switch (uExtIntType)
16831	{
16832	/*
16833	* Physical NMIs:
16834	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16835	*/
16836	case VMX_EXIT_INT_INFO_TYPE_NMI:
16837	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16838
16839	/*
16840	* Hardware exceptions,
16841	* Software exceptions,
16842	* Privileged software exceptions:
16843	* Figure out if the exception must be delivered to the guest or the nested-guest.
16844	*
16845	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16846	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16847	* length.
16848	*/
16849	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16850	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16851	{
16852	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16853	AssertRCReturn(rc, rc);
16854	RT_FALL_THRU();
16855	}
16856	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16857	{
16858	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16859	AssertRCReturn(rc, rc);
16860
16861	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
16862	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
16863	pVmxTransient->uExitIntErrorCode);
16864	if (fIntercept)
16865	{
16866	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16867	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16868	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16869	AssertRCReturn(rc, rc);
16870
16871	VMXVEXITINFO ExitInfo;
16872	RT_ZERO(ExitInfo);
16873	ExitInfo.uReason = pVmxTransient->uExitReason;
16874	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16875	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16876
16877	VMXVEXITEVENTINFO ExitEventInfo;
16878	RT_ZERO(ExitEventInfo);
16879	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16880	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16881	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16882	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16883
16884	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16885	}
16886
16887	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
16888	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16889
16890	/* If the guest hypervisor is not intercepting the exception, forward it to the guest. */
16891	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
16892	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
16893	return VINF_SUCCESS;
16894	}
16895
16896	/*
16897	* Software interrupts:
16898	* VM-exits cannot be caused by software interrupts.
16899	*
16900	* External interrupts:
16901	* This should only happen when "acknowledge external interrupts on VM-exit"
16902	* control is set. However, we never set this when executing a guest or
16903	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16904	* the guest.
16905	*/
16906	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16907	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16908	default:
16909	{
16910	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16911	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16912	}
16913	}
16914	}
16915
16916
16917	/**
16918	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16919	* Unconditional VM-exit.
16920	*/
16921	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16922	{
16923	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16924	return IEMExecVmxVmexitTripleFault(pVCpu);
16925	}
16926
16927
16928	/**
16929	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16930	*/
16931	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16932	{
16933	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16934
16935	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16936	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16937	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16938	}
16939
16940
16941	/**
16942	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16943	*/
16944	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16945	{
16946	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16947
16948	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16949	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16950	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16951	}
16952
16953
16954	/**
16955	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16956	* Unconditional VM-exit.
16957	*/
16958	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16959	{
16960	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16961
16962	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16963	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16964	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16965	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16966	AssertRCReturn(rc, rc);
16967
16968	VMXVEXITINFO ExitInfo;
16969	RT_ZERO(ExitInfo);
16970	ExitInfo.uReason = pVmxTransient->uExitReason;
16971	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16972	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16973
16974	VMXVEXITEVENTINFO ExitEventInfo;
16975	RT_ZERO(ExitEventInfo);
16976	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16977	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16978	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16979	}
16980
16981
16982	/**
16983	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16984	*/
16985	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16986	{
16987	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16988
16989	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16990	{
16991	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16992	AssertRCReturn(rc, rc);
16993	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16994	}
16995	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16996	}
16997
16998
16999	/**
17000	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
17001	*/
17002	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17003	{
17004	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17005
17006	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17007	{
17008	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17009	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17010	AssertRCReturn(rc, rc);
17011
17012	VMXVEXITINFO ExitInfo;
17013	RT_ZERO(ExitInfo);
17014	ExitInfo.uReason = pVmxTransient->uExitReason;
17015	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17016	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17017	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17018	}
17019	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
17020	}
17021
17022
17023	/**
17024	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
17025	*/
17026	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17027	{
17028	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17029
17030	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
17031	{
17032	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17033	AssertRCReturn(rc, rc);
17034	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17035	}
17036	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
17037	}
17038
17039
17040	/**
17041	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
17042	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
17043	*/
17044	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17045	{
17046	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17047
17048	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
17049	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
17050
17051	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17052	AssertRCReturn(rc, rc);
17053
17054	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
17055	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17056	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17057
17058	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
17059	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
17060	u64VmcsField &= UINT64_C(0xffffffff);
17061
17062	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
17063	{
17064	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17065	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17066	AssertRCReturn(rc, rc);
17067
17068	VMXVEXITINFO ExitInfo;
17069	RT_ZERO(ExitInfo);
17070	ExitInfo.uReason = pVmxTransient->uExitReason;
17071	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17072	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17073	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17074	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17075	}
17076
17077	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
17078	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
17079	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
17080	}
17081
17082
17083	/**
17084	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
17085	*/
17086	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17087	{
17088	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17089
17090	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17091	{
17092	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17093	AssertRCReturn(rc, rc);
17094	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17095	}
17096
17097	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
17098	}
17099
17100
17101	/**
17102	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
17103	* Conditional VM-exit.
17104	*/
17105	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17106	{
17107	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17108
17109	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17110	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17111	AssertRCReturn(rc, rc);
17112
17113	VBOXSTRICTRC rcStrict;
17114	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
17115	switch (uAccessType)
17116	{
17117	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
17118	{
17119	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17120	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17121	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17122	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17123
17124	bool fIntercept;
17125	switch (iCrReg)
17126	{
17127	case 0:
17128	case 4:
17129	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
17130	break;
17131
17132	case 3:
17133	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
17134	break;
17135
17136	case 8:
17137	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
17138	break;
17139
17140	default:
17141	fIntercept = false;
17142	break;
17143	}
17144	if (fIntercept)
17145	{
17146	VMXVEXITINFO ExitInfo;
17147	RT_ZERO(ExitInfo);
17148	ExitInfo.uReason = pVmxTransient->uExitReason;
17149	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17150	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17151	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17152	}
17153	else
17154	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17155	break;
17156	}
17157
17158	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
17159	{
17160	/*
17161	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
17162	* CR2 reads do not cause a VM-exit.
17163	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
17164	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
17165	*/
17166	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17167	if ( iCrReg == 3
17168	\|\| iCrReg == 8)
17169	{
17170	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
17171	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
17172	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
17173	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
17174	{
17175	VMXVEXITINFO ExitInfo;
17176	RT_ZERO(ExitInfo);
17177	ExitInfo.uReason = pVmxTransient->uExitReason;
17178	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17179	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17180	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17181	}
17182	else
17183	{
17184	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17185	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17186	}
17187	}
17188	else
17189	{
17190	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
17191	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
17192	}
17193	break;
17194	}
17195
17196	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
17197	{
17198	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
17199	Assert(pVmcsNstGst);
17200	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
17201	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
17202	if ( (uGstHostMask & X86_CR0_TS)
17203	&& (uReadShadow & X86_CR0_TS))
17204	{
17205	VMXVEXITINFO ExitInfo;
17206	RT_ZERO(ExitInfo);
17207	ExitInfo.uReason = pVmxTransient->uExitReason;
17208	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17209	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17210	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17211	}
17212	else
17213	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
17214	break;
17215	}
17216
17217	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
17218	{
17219	RTGCPTR GCPtrEffDst;
17220	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
17221	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
17222	if (fMemOperand)
17223	{
17224	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17225	AssertRCReturn(rc, rc);
17226	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
17227	}
17228	else
17229	GCPtrEffDst = NIL_RTGCPTR;
17230
17231	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
17232	{
17233	VMXVEXITINFO ExitInfo;
17234	RT_ZERO(ExitInfo);
17235	ExitInfo.uReason = pVmxTransient->uExitReason;
17236	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17237	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
17238	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17239	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17240	}
17241	else
17242	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
17243	break;
17244	}
17245
17246	default:
17247	{
17248	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
17249	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
17250	}
17251	}
17252
17253	if (rcStrict == VINF_IEM_RAISED_XCPT)
17254	{
17255	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
17256	rcStrict = VINF_SUCCESS;
17257	}
17258	return rcStrict;
17259	}
17260
17261
17262	/**
17263	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
17264	* Conditional VM-exit.
17265	*/
17266	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17267	{
17268	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17269
17270	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
17271	{
17272	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17273	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17274	AssertRCReturn(rc, rc);
17275
17276	VMXVEXITINFO ExitInfo;
17277	RT_ZERO(ExitInfo);
17278	ExitInfo.uReason = pVmxTransient->uExitReason;
17279	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17280	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17281	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17282	}
17283	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
17284	}
17285
17286
17287	/**
17288	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
17289	* Conditional VM-exit.
17290	*/
17291	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17292	{
17293	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17294
17295	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17296	AssertRCReturn(rc, rc);
17297
17298	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
17299	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
17300	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
17301
17302	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
17303	uint8_t const cbAccess = s_aIOSizes[uIOSize];
17304	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
17305	{
17306	/*
17307	* IN/OUT instruction:
17308	* - Provides VM-exit instruction length.
17309	*
17310	* INS/OUTS instruction:
17311	* - Provides VM-exit instruction length.
17312	* - Provides Guest-linear address.
17313	* - Optionally provides VM-exit instruction info (depends on CPU feature).
17314	*/
17315	PVM pVM = pVCpu->CTX_SUFF(pVM);
17316	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17317	AssertRCReturn(rc, rc);
17318
17319	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
17320	pVmxTransient->ExitInstrInfo.u = 0;
17321	pVmxTransient->uGuestLinearAddr = 0;
17322
17323	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
17324	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
17325	if (fIOString)
17326	{
17327	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17328	if (fVmxInsOutsInfo)
17329	{
17330	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
17331	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17332	}
17333	}
17334	AssertRCReturn(rc, rc);
17335
17336	VMXVEXITINFO ExitInfo;
17337	RT_ZERO(ExitInfo);
17338	ExitInfo.uReason = pVmxTransient->uExitReason;
17339	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17340	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17341	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17342	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
17343	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17344	}
17345	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
17346	}
17347
17348
17349	/**
17350	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
17351	*/
17352	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17353	{
17354	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17355
17356	uint32_t fMsrpm;
17357	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17358	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17359	else
17360	fMsrpm = VMXMSRPM_EXIT_RD;
17361
17362	if (fMsrpm & VMXMSRPM_EXIT_RD)
17363	{
17364	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17365	AssertRCReturn(rc, rc);
17366	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17367	}
17368	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
17369	}
17370
17371
17372	/**
17373	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
17374	*/
17375	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17376	{
17377	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17378
17379	uint32_t fMsrpm;
17380	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17381	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17382	else
17383	fMsrpm = VMXMSRPM_EXIT_WR;
17384
17385	if (fMsrpm & VMXMSRPM_EXIT_WR)
17386	{
17387	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17388	AssertRCReturn(rc, rc);
17389	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17390	}
17391	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17392	}
17393
17394
17395	/**
17396	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17397	*/
17398	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17399	{
17400	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17401
17402	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17403	{
17404	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17405	AssertRCReturn(rc, rc);
17406	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17407	}
17408	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17409	}
17410
17411
17412	/**
17413	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17414	* VM-exit.
17415	*/
17416	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17417	{
17418	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17419
17420	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17421	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17422	}
17423
17424
17425	/**
17426	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17427	*/
17428	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17429	{
17430	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17431
17432	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17433	{
17434	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17435	AssertRCReturn(rc, rc);
17436	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17437	}
17438	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17439	}
17440
17441
17442	/**
17443	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17444	*/
17445	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17446	{
17447	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17448
17449	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17450	* PAUSE when executing a nested-guest? If it does not, we would not need
17451	* to check for the intercepts here. Just call VM-exit... */
17452
17453	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17454	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17455	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17456	{
17457	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17458	AssertRCReturn(rc, rc);
17459	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17460	}
17461	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17462	}
17463
17464
17465	/**
17466	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17467	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17468	*/
17469	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17470	{
17471	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17472
17473	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17474	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17475	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17476	}
17477
17478
17479	/**
17480	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17481	* VM-exit.
17482	*/
17483	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17484	{
17485	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17486
17487	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17488	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17489	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17490	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17491	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17492	AssertRCReturn(rc, rc);
17493
17494	VMXVEXITINFO ExitInfo;
17495	RT_ZERO(ExitInfo);
17496	ExitInfo.uReason = pVmxTransient->uExitReason;
17497	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17498	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17499
17500	VMXVEXITEVENTINFO ExitEventInfo;
17501	RT_ZERO(ExitEventInfo);
17502	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17503	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17504	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17505	}
17506
17507
17508	/**
17509	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17510	* Conditional VM-exit.
17511	*/
17512	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17513	{
17514	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17515
17516	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17517	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17518	AssertRCReturn(rc, rc);
17519
17520	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17521	}
17522
17523
17524	/**
17525	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17526	* Conditional VM-exit.
17527	*/
17528	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17529	{
17530	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17531
17532	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17533	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17534	AssertRCReturn(rc, rc);
17535
17536	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17537	}
17538
17539
17540	/**
17541	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17542	*/
17543	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17544	{
17545	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17546
17547	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17548	{
17549	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17550	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17551	AssertRCReturn(rc, rc);
17552	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17553	}
17554	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17555	}
17556
17557
17558	/**
17559	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17560	*/
17561	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17562	{
17563	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17564
17565	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17566	{
17567	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17568	AssertRCReturn(rc, rc);
17569	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17570	}
17571	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17572	}
17573
17574
17575	/**
17576	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17577	*/
17578	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17579	{
17580	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17581
17582	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17583	{
17584	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17585	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17586	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17587	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17588	AssertRCReturn(rc, rc);
17589
17590	VMXVEXITINFO ExitInfo;
17591	RT_ZERO(ExitInfo);
17592	ExitInfo.uReason = pVmxTransient->uExitReason;
17593	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17594	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17595	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17596	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17597	}
17598	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17599	}
17600
17601
17602	/**
17603	* Nested-guest VM-exit handler for invalid-guest state
17604	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17605	*/
17606	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17607	{
17608	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17609
17610	/*
17611	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17612	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17613	* Handle it like it's in an invalid guest state of the outer guest.
17614	*
17615	* When the fast path is implemented, this should be changed to cause the corresponding
17616	* nested-guest VM-exit.
17617	*/
17618	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17619	}
17620
17621
17622	/**
17623	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17624	* and only provide the instruction length.
17625	*
17626	* Unconditional VM-exit.
17627	*/
17628	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17629	{
17630	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17631
17632	#ifdef VBOX_STRICT
17633	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17634	switch (pVmxTransient->uExitReason)
17635	{
17636	case VMX_EXIT_ENCLS:
17637	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17638	break;
17639
17640	case VMX_EXIT_VMFUNC:
17641	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17642	break;
17643	}
17644	#endif
17645
17646	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17647	AssertRCReturn(rc, rc);
17648	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17649	}
17650
17651
17652	/**
17653	* Nested-guest VM-exit handler for instructions that provide instruction length as
17654	* well as more information.
17655	*
17656	* Unconditional VM-exit.
17657	*/
17658	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17659	{
17660	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17661
17662	#ifdef VBOX_STRICT
17663	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17664	switch (pVmxTransient->uExitReason)
17665	{
17666	case VMX_EXIT_GDTR_IDTR_ACCESS:
17667	case VMX_EXIT_LDTR_TR_ACCESS:
17668	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17669	break;
17670
17671	case VMX_EXIT_RDRAND:
17672	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17673	break;
17674
17675	case VMX_EXIT_RDSEED:
17676	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17677	break;
17678
17679	case VMX_EXIT_XSAVES:
17680	case VMX_EXIT_XRSTORS:
17681	/** @todo NSTVMX: Verify XSS-bitmap. */
17682	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17683	break;
17684
17685	case VMX_EXIT_UMWAIT:
17686	case VMX_EXIT_TPAUSE:
17687	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17688	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17689	break;
17690	}
17691	#endif
17692
17693	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17694	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17695	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17696	AssertRCReturn(rc, rc);
17697
17698	VMXVEXITINFO ExitInfo;
17699	RT_ZERO(ExitInfo);
17700	ExitInfo.uReason = pVmxTransient->uExitReason;
17701	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17702	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17703	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17704	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17705	}
17706
17707	/** @} */
17708	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17709

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79660

Download in other formats: