HMVMXR0.cpp@ 79564

Last change on this file since 79564 was 79564, checked in by vboxsync, 5 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 nit.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 732.8 KB

Line
1	/* $Id: HMVMXR0.cpp 79564 2019-07-06 06:57:37Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#ifdef VBOX_WITH_REM
39	# include <VBox/vmm/rem.h>
40	#endif
41	#include "HMInternal.h"
42	#include <VBox/vmm/vm.h>
43	#include <VBox/vmm/hmvmxinline.h>
44	#include "HMVMXR0.h"
45	#include "dtrace/VBoxVMM.h"
46
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
51	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
52	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
53	# define HMVMX_ALWAYS_TRAP_PF
54	# define HMVMX_ALWAYS_FLUSH_TLB
55	# define HMVMX_ALWAYS_SWAP_EFER
56	#endif
57
58
59	/*********************************************************************************************************************************
60	* Defined Constants And Macros *
61	*********************************************************************************************************************************/
62	/** Use the function table. */
63	#define HMVMX_USE_FUNCTION_TABLE
64
65	/** Determine which tagged-TLB flush handler to use. */
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71	/** @name HMVMX_READ_XXX
72	* Flags to skip redundant reads of some common VMCS fields that are not part of
73	* the guest-CPU or VCPU state but are needed while handling VM-exits.
74	*/
75	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
76	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
77	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
78	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
79	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
80	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
81	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
82	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
83	/** @} */
84
85	/**
86	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
87	* guest using hardware-assisted VMX.
88	*
89	* This excludes state like GPRs (other than RSP) which are always are
90	* swapped and restored across the world-switch and also registers like EFER,
91	* MSR which cannot be modified by the guest without causing a VM-exit.
92	*/
93	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
94	\| CPUMCTX_EXTRN_RFLAGS \
95	\| CPUMCTX_EXTRN_RSP \
96	\| CPUMCTX_EXTRN_SREG_MASK \
97	\| CPUMCTX_EXTRN_TABLE_MASK \
98	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
99	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
100	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
101	\| CPUMCTX_EXTRN_TSC_AUX \
102	\| CPUMCTX_EXTRN_OTHER_MSRS \
103	\| CPUMCTX_EXTRN_CR0 \
104	\| CPUMCTX_EXTRN_CR3 \
105	\| CPUMCTX_EXTRN_CR4 \
106	\| CPUMCTX_EXTRN_DR7 \
107	\| CPUMCTX_EXTRN_HWVIRT \
108	\| CPUMCTX_EXTRN_HM_VMX_MASK)
109
110	/**
111	* Exception bitmap mask for real-mode guests (real-on-v86).
112	*
113	* We need to intercept all exceptions manually except:
114	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
115	* due to bugs in Intel CPUs.
116	* - \#PF need not be intercepted even in real-mode if we have nested paging
117	* support.
118	*/
119	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
120	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
121	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
122	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
123	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
124	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
125	\| RT_BIT(X86_XCPT_XF))
126
127	/** Maximum VM-instruction error number. */
128	#define HMVMX_INSTR_ERROR_MAX 28
129
130	/** Profiling macro. */
131	#ifdef HM_PROFILE_EXIT_DISPATCH
132	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
133	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
134	#else
135	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
136	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
137	#endif
138
139	/** Assert that preemption is disabled or covered by thread-context hooks. */
140	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
141	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
142
143	/** Assert that we haven't migrated CPUs when thread-context hooks are not
144	* used. */
145	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
146	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
147	("Illegal migration! Entered on CPU %u Current %u\n", \
148	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
149
150	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
151	* context. */
152	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
153	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
154	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
155
156
157	/*********************************************************************************************************************************
158	* Structures and Typedefs *
159	*********************************************************************************************************************************/
160	/**
161	* VMX per-VCPU transient state.
162	*
163	* A state structure for holding miscellaneous information across
164	* VMX non-root operation and restored after the transition.
165	*/
166	typedef struct VMXTRANSIENT
167	{
168	/** The host's rflags/eflags. */
169	RTCCUINTREG fEFlags;
170	#if HC_ARCH_BITS == 32
171	uint32_t u32Alignment0;
172	#endif
173	/** The guest's TPR value used for TPR shadowing. */
174	uint8_t u8GuestTpr;
175	/** Alignment. */
176	uint8_t abAlignment0[7];
177
178	/** The basic VM-exit reason. */
179	uint16_t uExitReason;
180	/** Alignment. */
181	uint16_t u16Alignment0;
182	/** The VM-exit interruption error code. */
183	uint32_t uExitIntErrorCode;
184	/** The VM-exit exit code qualification. */
185	uint64_t uExitQual;
186	/** The Guest-linear address. */
187	uint64_t uGuestLinearAddr;
188
189	/** The VM-exit interruption-information field. */
190	uint32_t uExitIntInfo;
191	/** The VM-exit instruction-length field. */
192	uint32_t cbInstr;
193	/** The VM-exit instruction-information field. */
194	VMXEXITINSTRINFO ExitInstrInfo;
195	/** Whether the VM-entry failed or not. */
196	bool fVMEntryFailed;
197	/** Whether we are currently executing a nested-guest. */
198	bool fIsNestedGuest;
199	/** Alignment. */
200	uint8_t abAlignment1[2];
201
202	/** The VM-entry interruption-information field. */
203	uint32_t uEntryIntInfo;
204	/** The VM-entry exception error code field. */
205	uint32_t uEntryXcptErrorCode;
206	/** The VM-entry instruction length field. */
207	uint32_t cbEntryInstr;
208
209	/** IDT-vectoring information field. */
210	uint32_t uIdtVectoringInfo;
211	/** IDT-vectoring error code. */
212	uint32_t uIdtVectoringErrorCode;
213
214	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
215	uint32_t fVmcsFieldsRead;
216
217	/** Whether the guest debug state was active at the time of VM-exit. */
218	bool fWasGuestDebugStateActive;
219	/** Whether the hyper debug state was active at the time of VM-exit. */
220	bool fWasHyperDebugStateActive;
221	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
222	bool fUpdatedTscOffsettingAndPreemptTimer;
223	/** Whether the VM-exit was caused by a page-fault during delivery of a
224	* contributory exception or a page-fault. */
225	bool fVectoringDoublePF;
226	/** Whether the VM-exit was caused by a page-fault during delivery of an
227	* external interrupt or NMI. */
228	bool fVectoringPF;
229	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
230	* area after VM-exit. */
231	bool fRemoveTscAuxMsr;
232	bool afAlignment0[2];
233
234	/** The VMCS info. object. */
235	PVMXVMCSINFO pVmcsInfo;
236	} VMXTRANSIENT;
237	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
238	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
239	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
240	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
241	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
242	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
243	/** Pointer to VMX transient state. */
244	typedef VMXTRANSIENT *PVMXTRANSIENT;
245	/** Pointer to a const VMX transient state. */
246	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
247
248	/**
249	* Memory operand read or write access.
250	*/
251	typedef enum VMXMEMACCESS
252	{
253	VMXMEMACCESS_READ = 0,
254	VMXMEMACCESS_WRITE = 1
255	} VMXMEMACCESS;
256
257	/**
258	* VMX VM-exit handler.
259	*
260	* @returns Strict VBox status code (i.e. informational status codes too).
261	* @param pVCpu The cross context virtual CPU structure.
262	* @param pVmxTransient The VMX-transient structure.
263	*/
264	#ifndef HMVMX_USE_FUNCTION_TABLE
265	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
266	#else
267	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
268	/** Pointer to VM-exit handler. */
269	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
270	#endif
271
272	/**
273	* VMX VM-exit handler, non-strict status code.
274	*
275	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
276	*
277	* @returns VBox status code, no informational status code returned.
278	* @param pVCpu The cross context virtual CPU structure.
279	* @param pVmxTransient The VMX-transient structure.
280	*
281	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
282	* use of that status code will be replaced with VINF_EM_SOMETHING
283	* later when switching over to IEM.
284	*/
285	#ifndef HMVMX_USE_FUNCTION_TABLE
286	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
287	#else
288	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
289	#endif
290
291
292	/*********************************************************************************************************************************
293	* Internal Functions *
294	*********************************************************************************************************************************/
295	#ifndef HMVMX_USE_FUNCTION_TABLE
296	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
297	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
298	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
299	#else
300	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
301	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
302	#endif
303	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
304	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
305	#endif
306
307	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
308	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
309	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
310	#endif
311
312	/** @name VM-exit handler prototypes.
313	* @{
314	*/
315	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
316	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
317	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
318	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
319	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
320	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
321	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
322	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
323	static FNVMXEXITHANDLER hmR0VmxExitHlt;
324	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
325	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
326	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
327	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
328	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
329	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
330	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
331	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
332	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
333	static FNVMXEXITHANDLER hmR0VmxExitVmread;
334	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
335	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
336	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
337	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
338	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
339	#endif
340	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
341	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
342	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
343	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
344	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
345	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
346	static FNVMXEXITHANDLER hmR0VmxExitMwait;
347	static FNVMXEXITHANDLER hmR0VmxExitMtf;
348	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
349	static FNVMXEXITHANDLER hmR0VmxExitPause;
350	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
351	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
352	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
353	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
354	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
355	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
356	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
357	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
358	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
359	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
360	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
361	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
362	/** @} */
363
364	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
365	/** @name Nested-guest VM-exit handler prototypes.
366	* @{
367	*/
368	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
369	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
371	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
372	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
373	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
374	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
375	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
376	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
377	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
378	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
379	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
380	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
381	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
382	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
383	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
384	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
385	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
386	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
388	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
389	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
390	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
391	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
392	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
393	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
394	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
395	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
396	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
397	/** @} */
398	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
399
400
401	/*********************************************************************************************************************************
402	* Global Variables *
403	*********************************************************************************************************************************/
404	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
405	/**
406	* Array of all VMCS fields.
407	* Any fields added to the VT-x spec. should be added here.
408	*
409	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
410	* of nested-guests.
411	*/
412	static const uint32_t g_aVmcsFields[] =
413	{
414	/* 16-bit control fields. */
415	VMX_VMCS16_VPID,
416	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
417	VMX_VMCS16_EPTP_INDEX,
418
419	/* 16-bit guest-state fields. */
420	VMX_VMCS16_GUEST_ES_SEL,
421	VMX_VMCS16_GUEST_CS_SEL,
422	VMX_VMCS16_GUEST_SS_SEL,
423	VMX_VMCS16_GUEST_DS_SEL,
424	VMX_VMCS16_GUEST_FS_SEL,
425	VMX_VMCS16_GUEST_GS_SEL,
426	VMX_VMCS16_GUEST_LDTR_SEL,
427	VMX_VMCS16_GUEST_TR_SEL,
428	VMX_VMCS16_GUEST_INTR_STATUS,
429	VMX_VMCS16_GUEST_PML_INDEX,
430
431	/* 16-bits host-state fields. */
432	VMX_VMCS16_HOST_ES_SEL,
433	VMX_VMCS16_HOST_CS_SEL,
434	VMX_VMCS16_HOST_SS_SEL,
435	VMX_VMCS16_HOST_DS_SEL,
436	VMX_VMCS16_HOST_FS_SEL,
437	VMX_VMCS16_HOST_GS_SEL,
438	VMX_VMCS16_HOST_TR_SEL,
439
440	/* 64-bit control fields. */
441	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
442	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
443	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
444	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
445	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
446	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
447	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
448	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
449	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
450	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
451	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
452	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
453	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
454	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
455	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
456	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
457	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
458	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
459	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
460	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
461	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
462	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
463	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
464	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
465	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
466	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
467	VMX_VMCS64_CTRL_EPTP_FULL,
468	VMX_VMCS64_CTRL_EPTP_HIGH,
469	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
470	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
471	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
472	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
473	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
474	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
475	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
476	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
477	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
478	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
479	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
480	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
481	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
482	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
483	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
484	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
485	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
486	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
487	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
488	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
489	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
490	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
491
492	/* 64-bit read-only data fields. */
493	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
494	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
495
496	/* 64-bit guest-state fields. */
497	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
498	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
499	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
500	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
501	VMX_VMCS64_GUEST_PAT_FULL,
502	VMX_VMCS64_GUEST_PAT_HIGH,
503	VMX_VMCS64_GUEST_EFER_FULL,
504	VMX_VMCS64_GUEST_EFER_HIGH,
505	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
506	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
507	VMX_VMCS64_GUEST_PDPTE0_FULL,
508	VMX_VMCS64_GUEST_PDPTE0_HIGH,
509	VMX_VMCS64_GUEST_PDPTE1_FULL,
510	VMX_VMCS64_GUEST_PDPTE1_HIGH,
511	VMX_VMCS64_GUEST_PDPTE2_FULL,
512	VMX_VMCS64_GUEST_PDPTE2_HIGH,
513	VMX_VMCS64_GUEST_PDPTE3_FULL,
514	VMX_VMCS64_GUEST_PDPTE3_HIGH,
515	VMX_VMCS64_GUEST_BNDCFGS_FULL,
516	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
517
518	/* 64-bit host-state fields. */
519	VMX_VMCS64_HOST_PAT_FULL,
520	VMX_VMCS64_HOST_PAT_HIGH,
521	VMX_VMCS64_HOST_EFER_FULL,
522	VMX_VMCS64_HOST_EFER_HIGH,
523	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
524	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
525
526	/* 32-bit control fields. */
527	VMX_VMCS32_CTRL_PIN_EXEC,
528	VMX_VMCS32_CTRL_PROC_EXEC,
529	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
530	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
531	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
532	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
533	VMX_VMCS32_CTRL_EXIT,
534	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
535	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
536	VMX_VMCS32_CTRL_ENTRY,
537	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
538	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
539	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
540	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
541	VMX_VMCS32_CTRL_TPR_THRESHOLD,
542	VMX_VMCS32_CTRL_PROC_EXEC2,
543	VMX_VMCS32_CTRL_PLE_GAP,
544	VMX_VMCS32_CTRL_PLE_WINDOW,
545
546	/* 32-bits read-only fields. */
547	VMX_VMCS32_RO_VM_INSTR_ERROR,
548	VMX_VMCS32_RO_EXIT_REASON,
549	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
550	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
551	VMX_VMCS32_RO_IDT_VECTORING_INFO,
552	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
553	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
554	VMX_VMCS32_RO_EXIT_INSTR_INFO,
555
556	/* 32-bit guest-state fields. */
557	VMX_VMCS32_GUEST_ES_LIMIT,
558	VMX_VMCS32_GUEST_CS_LIMIT,
559	VMX_VMCS32_GUEST_SS_LIMIT,
560	VMX_VMCS32_GUEST_DS_LIMIT,
561	VMX_VMCS32_GUEST_FS_LIMIT,
562	VMX_VMCS32_GUEST_GS_LIMIT,
563	VMX_VMCS32_GUEST_LDTR_LIMIT,
564	VMX_VMCS32_GUEST_TR_LIMIT,
565	VMX_VMCS32_GUEST_GDTR_LIMIT,
566	VMX_VMCS32_GUEST_IDTR_LIMIT,
567	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
568	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
569	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
570	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
571	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
572	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
573	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
574	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
575	VMX_VMCS32_GUEST_INT_STATE,
576	VMX_VMCS32_GUEST_ACTIVITY_STATE,
577	VMX_VMCS32_GUEST_SMBASE,
578	VMX_VMCS32_GUEST_SYSENTER_CS,
579	VMX_VMCS32_PREEMPT_TIMER_VALUE,
580
581	/* 32-bit host-state fields. */
582	VMX_VMCS32_HOST_SYSENTER_CS,
583
584	/* Natural-width control fields. */
585	VMX_VMCS_CTRL_CR0_MASK,
586	VMX_VMCS_CTRL_CR4_MASK,
587	VMX_VMCS_CTRL_CR0_READ_SHADOW,
588	VMX_VMCS_CTRL_CR4_READ_SHADOW,
589	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
590	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
591	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
592	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
593
594	/* Natural-width read-only data fields. */
595	VMX_VMCS_RO_EXIT_QUALIFICATION,
596	VMX_VMCS_RO_IO_RCX,
597	VMX_VMCS_RO_IO_RSI,
598	VMX_VMCS_RO_IO_RDI,
599	VMX_VMCS_RO_IO_RIP,
600	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
601
602	/* Natural-width guest-state field */
603	VMX_VMCS_GUEST_CR0,
604	VMX_VMCS_GUEST_CR3,
605	VMX_VMCS_GUEST_CR4,
606	VMX_VMCS_GUEST_ES_BASE,
607	VMX_VMCS_GUEST_CS_BASE,
608	VMX_VMCS_GUEST_SS_BASE,
609	VMX_VMCS_GUEST_DS_BASE,
610	VMX_VMCS_GUEST_FS_BASE,
611	VMX_VMCS_GUEST_GS_BASE,
612	VMX_VMCS_GUEST_LDTR_BASE,
613	VMX_VMCS_GUEST_TR_BASE,
614	VMX_VMCS_GUEST_GDTR_BASE,
615	VMX_VMCS_GUEST_IDTR_BASE,
616	VMX_VMCS_GUEST_DR7,
617	VMX_VMCS_GUEST_RSP,
618	VMX_VMCS_GUEST_RIP,
619	VMX_VMCS_GUEST_RFLAGS,
620	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
621	VMX_VMCS_GUEST_SYSENTER_ESP,
622	VMX_VMCS_GUEST_SYSENTER_EIP,
623
624	/* Natural-width host-state fields */
625	VMX_VMCS_HOST_CR0,
626	VMX_VMCS_HOST_CR3,
627	VMX_VMCS_HOST_CR4,
628	VMX_VMCS_HOST_FS_BASE,
629	VMX_VMCS_HOST_GS_BASE,
630	VMX_VMCS_HOST_TR_BASE,
631	VMX_VMCS_HOST_GDTR_BASE,
632	VMX_VMCS_HOST_IDTR_BASE,
633	VMX_VMCS_HOST_SYSENTER_ESP,
634	VMX_VMCS_HOST_SYSENTER_EIP,
635	VMX_VMCS_HOST_RSP,
636	VMX_VMCS_HOST_RIP
637	};
638	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
639
640	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
641	static const uint32_t g_aVmcsCacheSegBase[] =
642	{
643	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
644	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
645	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
646	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
647	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
648	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
649	};
650	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
651	#endif
652	static const uint32_t g_aVmcsSegBase[] =
653	{
654	VMX_VMCS_GUEST_ES_BASE,
655	VMX_VMCS_GUEST_CS_BASE,
656	VMX_VMCS_GUEST_SS_BASE,
657	VMX_VMCS_GUEST_DS_BASE,
658	VMX_VMCS_GUEST_FS_BASE,
659	VMX_VMCS_GUEST_GS_BASE
660	};
661	static const uint32_t g_aVmcsSegSel[] =
662	{
663	VMX_VMCS16_GUEST_ES_SEL,
664	VMX_VMCS16_GUEST_CS_SEL,
665	VMX_VMCS16_GUEST_SS_SEL,
666	VMX_VMCS16_GUEST_DS_SEL,
667	VMX_VMCS16_GUEST_FS_SEL,
668	VMX_VMCS16_GUEST_GS_SEL
669	};
670	static const uint32_t g_aVmcsSegLimit[] =
671	{
672	VMX_VMCS32_GUEST_ES_LIMIT,
673	VMX_VMCS32_GUEST_CS_LIMIT,
674	VMX_VMCS32_GUEST_SS_LIMIT,
675	VMX_VMCS32_GUEST_DS_LIMIT,
676	VMX_VMCS32_GUEST_FS_LIMIT,
677	VMX_VMCS32_GUEST_GS_LIMIT
678	};
679	static const uint32_t g_aVmcsSegAttr[] =
680	{
681	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
682	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
683	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
684	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
685	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
686	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
687	};
688	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
689	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
690	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
691	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
692
693	#ifdef HMVMX_USE_FUNCTION_TABLE
694	/**
695	* VMX_EXIT dispatch table.
696	*/
697	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
698	{
699	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
700	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
701	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
702	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
703	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
704	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
705	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
706	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
707	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
708	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
709	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
710	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
711	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
712	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
713	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
714	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
715	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
716	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
717	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
718	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
719	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
720	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
721	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
722	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
723	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
724	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
725	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
726	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
727	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
728	#else
729	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
730	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
731	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
732	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
733	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
734	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
735	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
736	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
737	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
738	#endif
739	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
740	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
741	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
742	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
743	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
744	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
745	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
746	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
747	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
748	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
749	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
750	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
751	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
752	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
753	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
754	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
755	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
756	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
757	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
758	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
759	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
760	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
761	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
762	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
763	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
764	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
765	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
766	#else
767	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
768	#endif
769	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
770	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
771	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
772	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
773	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
774	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
775	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
776	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
777	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
778	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
779	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
780	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
781	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
782	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
783	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
784	};
785	#endif /* HMVMX_USE_FUNCTION_TABLE */
786
787	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
788	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
789	{
790	/* 0 */ "(Not Used)",
791	/* 1 */ "VMCALL executed in VMX root operation.",
792	/* 2 */ "VMCLEAR with invalid physical address.",
793	/* 3 */ "VMCLEAR with VMXON pointer.",
794	/* 4 */ "VMLAUNCH with non-clear VMCS.",
795	/* 5 */ "VMRESUME with non-launched VMCS.",
796	/* 6 */ "VMRESUME after VMXOFF",
797	/* 7 */ "VM-entry with invalid control fields.",
798	/* 8 */ "VM-entry with invalid host state fields.",
799	/* 9 */ "VMPTRLD with invalid physical address.",
800	/* 10 */ "VMPTRLD with VMXON pointer.",
801	/* 11 */ "VMPTRLD with incorrect revision identifier.",
802	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
803	/* 13 */ "VMWRITE to read-only VMCS component.",
804	/* 14 */ "(Not Used)",
805	/* 15 */ "VMXON executed in VMX root operation.",
806	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
807	/* 17 */ "VM-entry with non-launched executing VMCS.",
808	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
809	/* 19 */ "VMCALL with non-clear VMCS.",
810	/* 20 */ "VMCALL with invalid VM-exit control fields.",
811	/* 21 */ "(Not Used)",
812	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
813	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
814	/* 24 */ "VMCALL with invalid SMM-monitor features.",
815	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
816	/* 26 */ "VM-entry with events blocked by MOV SS.",
817	/* 27 */ "(Not Used)",
818	/* 28 */ "Invalid operand to INVEPT/INVVPID."
819	};
820	#endif /* VBOX_STRICT && LOG_ENABLED */
821
822
823	/**
824	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
825	*
826	* Any bit set in this mask is owned by the host/hypervisor and would cause a
827	* VM-exit when modified by the guest.
828	*
829	* @returns The static CR0 guest/host mask.
830	* @param pVCpu The cross context virtual CPU structure.
831	*/
832	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
833	{
834	/*
835	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
836	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
837	*/
838	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
839	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
840	* and @bugref{6944}. */
841	PVM pVM = pVCpu->CTX_SUFF(pVM);
842	return ( X86_CR0_PE
843	\| X86_CR0_NE
844	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
845	\| X86_CR0_PG
846	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
847	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
848	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
849	}
850
851
852	/**
853	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
854	*
855	* Any bit set in this mask is owned by the host/hypervisor and would cause a
856	* VM-exit when modified by the guest.
857	*
858	* @returns The static CR4 guest/host mask.
859	* @param pVCpu The cross context virtual CPU structure.
860	*/
861	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
862	{
863	/*
864	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
865	* these bits are reserved on hardware that does not support them. Since the
866	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
867	* these bits and handle it depending on whether we expose them to the guest.
868	*/
869	PVM pVM = pVCpu->CTX_SUFF(pVM);
870	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
871	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
872	return ( X86_CR4_VMXE
873	\| X86_CR4_VME
874	\| X86_CR4_PAE
875	\| X86_CR4_PGE
876	\| X86_CR4_PSE
877	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
878	\| (fPcid ? X86_CR4_PCIDE : 0));
879	}
880
881
882	/**
883	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
884	* area.
885	*
886	* @returns @c true if it's different, @c false otherwise.
887	* @param pVmcsInfo The VMCS info. object.
888	*/
889	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
890	{
891	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
892	&& pVmcsInfo->pvGuestMsrStore);
893	}
894
895
896	/**
897	* Checks whether one of the given Pin-based VM-execution controls are set.
898	*
899	* @returns @c true if set, @c false otherwise.
900	* @param pVCpu The cross context virtual CPU structure.
901	* @param pVmxTransient The VMX-transient structure.
902	* @param uPinCtls The Pin-based VM-execution controls to check.
903	*
904	* @remarks This will not check merged controls when executing a nested-guest
905	* but the original control specified by the guest hypervisor.
906	*/
907	static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
908	{
909	if (!pVmxTransient->fIsNestedGuest)
910	{
911	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
912	return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
913	}
914	return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
915	}
916
917
918	#if 0
919	/**
920	* Checks whether one of the given Processor-based VM-execution controls are set.
921	*
922	* @returns @c true if set, @c false otherwise.
923	* @param pVCpu The cross context virtual CPU structure.
924	* @param pVmxTransient The VMX-transient structure.
925	* @param uProcCtls The Processor-based VM-execution controls to check.
926	*
927	* @remarks This will not check merged controls when executing a nested-guest
928	* but the original control specified by the guest hypervisor.
929	*/
930	static bool hmR0VmxIsProcCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
931	{
932	if (!pVmxTransient->fIsNestedGuest)
933	{
934	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
935	return RT_BOOL(pVmcsInfo->u32ProcCtls & uProcCtls);
936	}
937	return CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
938	}
939
940
941	/**
942	* Checks whether one of the given Secondary Processor-based VM-execution controls
943	* are set.
944	*
945	* @returns @c true if set, @c false otherwise.
946	* @param pVCpu The cross context virtual CPU structure.
947	* @param pVmxTransient The VMX-transient structure.
948	* @param uProcCtls2 The Secondary Processor-based VM-execution controls to
949	* check.
950	*
951	* @remarks This will not check merged controls when executing a nested-guest
952	* but the original control specified by the guest hypervisor.
953	*/
954	static bool hmR0VmxIsProcCtls2Set(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls2)
955	{
956	if (!pVmxTransient->fIsNestedGuest)
957	{
958	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
959	return RT_BOOL(pVmcsInfo->u32ProcCtls2 & uProcCtls2);
960	}
961	return CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls2);
962	}
963
964
965	/**
966	* Checks whether one of the given VM-entry controls are set.
967	*
968	* @returns @c true if set, @c false otherwise.
969	* @param pVCpu The cross context virtual CPU structure.
970	* @param pVmxTransient The VMX-transient structure.
971	* @param uEntryCtls The VM-entry controls to check.
972	*
973	* @remarks This will not check merged controls when executing a nested-guest
974	* but the original control specified by the guest hypervisor.
975	*/
976	static bool hmR0VmxIsEntryCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uEntryCtls)
977	{
978	if (!pVmxTransient->fIsNestedGuest)
979	{
980	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
981	return RT_BOOL(pVmcsInfo->u32EntryCtls & uEntryCtls);
982	}
983	return CPUMIsGuestVmxEntryCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uEntryCtls);
984	}
985
986
987	/**
988	* Checks whether one of the given VM-exit controls are set.
989	*
990	* @returns @c true if set, @c false otherwise.
991	* @param pVCpu The cross context virtual CPU structure.
992	* @param pVmxTransient The VMX-transient structure.
993	* @param uExitCtls The VM-exit controls to check.
994	*
995	* @remarks This will not check merged controls when executing a nested-guest
996	* but the original control specified by the guest hypervisor.
997	*/
998	static bool hmR0VmxIsExitCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uExitCtls)
999	{
1000	if (!pVmxTransient->fIsNestedGuest)
1001	{
1002	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1003	return RT_BOOL(pVmcsInfo->u32ExitCtls & uExitCtls);
1004	}
1005	return CPUMIsGuestVmxExitCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uExitCtls);
1006	}
1007	#endif
1008
1009
1010	/**
1011	* Adds one or more exceptions to the exception bitmap and commits it to the current
1012	* VMCS.
1013	*
1014	* @returns VBox status code.
1015	* @param pVmxTransient The VMX-transient structure.
1016	* @param uXcptMask The exception(s) to add.
1017	*/
1018	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1019	{
1020	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1021	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1022	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1023	{
1024	uXcptBitmap \|= uXcptMask;
1025	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1026	AssertRCReturn(rc, rc);
1027	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1028	}
1029	return VINF_SUCCESS;
1030	}
1031
1032
1033	/**
1034	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1035	*
1036	* @returns VBox status code.
1037	* @param pVmxTransient The VMX-transient structure.
1038	* @param uXcpt The exception to add.
1039	*/
1040	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1041	{
1042	Assert(uXcpt <= X86_XCPT_LAST);
1043	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1044	}
1045
1046
1047	/**
1048	* Remove one or more exceptions from the exception bitmap and commits it to the
1049	* current VMCS.
1050	*
1051	* This takes care of not removing the exception intercept if a nested-guest
1052	* requires the exception to be intercepted.
1053	*
1054	* @returns VBox status code.
1055	* @param pVCpu The cross context virtual CPU structure.
1056	* @param pVmxTransient The VMX-transient structure.
1057	* @param uXcptMask The exception(s) to remove.
1058	*/
1059	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1060	{
1061	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1062	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1063	if (u32XcptBitmap & uXcptMask)
1064	{
1065	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1066	if (!pVmxTransient->fIsNestedGuest)
1067	{ /* likely */ }
1068	else
1069	{
1070	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1071	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1072	}
1073	#endif
1074	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1075	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1076	\| RT_BIT(X86_XCPT_DE)
1077	\| RT_BIT(X86_XCPT_NM)
1078	\| RT_BIT(X86_XCPT_TS)
1079	\| RT_BIT(X86_XCPT_UD)
1080	\| RT_BIT(X86_XCPT_NP)
1081	\| RT_BIT(X86_XCPT_SS)
1082	\| RT_BIT(X86_XCPT_GP)
1083	\| RT_BIT(X86_XCPT_PF)
1084	\| RT_BIT(X86_XCPT_MF));
1085	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1086	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1087	#endif
1088	if (uXcptMask)
1089	{
1090	/* Validate we are not removing any essential exception intercepts. */
1091	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1092	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1093	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1094
1095	/* Remove it from the exception bitmap. */
1096	u32XcptBitmap &= ~uXcptMask;
1097
1098	/* Commit and update the cache if necessary. */
1099	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1100	{
1101	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1102	AssertRCReturn(rc, rc);
1103	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1104	}
1105	}
1106	}
1107	return VINF_SUCCESS;
1108	}
1109
1110
1111	/**
1112	* Remove an exceptions from the exception bitmap and commits it to the current
1113	* VMCS.
1114	*
1115	* @returns VBox status code.
1116	* @param pVCpu The cross context virtual CPU structure.
1117	* @param pVmxTransient The VMX-transient structure.
1118	* @param uXcpt The exception to remove.
1119	*/
1120	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1121	{
1122	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1123	}
1124
1125
1126	/**
1127	* Loads the VMCS specified by the VMCS info. object.
1128	*
1129	* @returns VBox status code.
1130	* @param pVmcsInfo The VMCS info. object.
1131	*
1132	* @remarks Can be called with interrupts disabled.
1133	*/
1134	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1135	{
1136	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1137	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1138
1139	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1140	if (RT_SUCCESS(rc))
1141	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1142	return rc;
1143	}
1144
1145
1146	/**
1147	* Clears the VMCS specified by the VMCS info. object.
1148	*
1149	* @returns VBox status code.
1150	* @param pVmcsInfo The VMCS info. object.
1151	*
1152	* @remarks Can be called with interrupts disabled.
1153	*/
1154	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1155	{
1156	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1157	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1158
1159	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1160	if (RT_SUCCESS(rc))
1161	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1162	return rc;
1163	}
1164
1165
1166	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1167	/**
1168	* Loads the shadow VMCS specified by the VMCS info. object.
1169	*
1170	* @returns VBox status code.
1171	* @param pVmcsInfo The VMCS info. object.
1172	*
1173	* @remarks Can be called with interrupts disabled.
1174	*/
1175	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1176	{
1177	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1178	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1179
1180	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1181	if (RT_SUCCESS(rc))
1182	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1183	return rc;
1184	}
1185
1186
1187	/**
1188	* Clears the shadow VMCS specified by the VMCS info. object.
1189	*
1190	* @returns VBox status code.
1191	* @param pVmcsInfo The VMCS info. object.
1192	*
1193	* @remarks Can be called with interrupts disabled.
1194	*/
1195	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1196	{
1197	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1198	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1199
1200	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1201	if (RT_SUCCESS(rc))
1202	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1203	return rc;
1204	}
1205
1206
1207	/**
1208	* Switches from and to the specified VMCSes.
1209	*
1210	* @returns VBox status code.
1211	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1212	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1213	*
1214	* @remarks Called with interrupts disabled.
1215	*/
1216	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1217	{
1218	/*
1219	* Clear the VMCS we are switching out if it has not already been cleared.
1220	* This will sync any CPU internal data back to the VMCS.
1221	*/
1222	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1223	{
1224	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1225	if (RT_SUCCESS(rc))
1226	{
1227	/*
1228	* The shadow VMCS, if any, would not be active at this point since we
1229	* would have cleared it while importing the virtual hardware-virtualization
1230	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1231	* clear the shadow VMCS here, just assert for safety.
1232	*/
1233	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1234	}
1235	else
1236	return rc;
1237	}
1238
1239	/*
1240	* Clear the VMCS we are switching to if it has not already been cleared.
1241	* This will initialize the VMCS launch state to "clear" required for loading it.
1242	*
1243	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1244	*/
1245	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1246	{
1247	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1248	if (RT_SUCCESS(rc))
1249	{ /* likely */ }
1250	else
1251	return rc;
1252	}
1253
1254	/*
1255	* Finally, load the VMCS we are switching to.
1256	*/
1257	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1258	}
1259
1260
1261	/**
1262	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1263	* caller.
1264	*
1265	* @returns VBox status code.
1266	* @param pVCpu The cross context virtual CPU structure.
1267	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1268	* true) or guest VMCS (pass false).
1269	*/
1270	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPU pVCpu, bool fSwitchToNstGstVmcs)
1271	{
1272	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1273	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1274
1275	PVMXVMCSINFO pVmcsInfoFrom;
1276	PVMXVMCSINFO pVmcsInfoTo;
1277	if (fSwitchToNstGstVmcs)
1278	{
1279	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1280	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1281	}
1282	else
1283	{
1284	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1285	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1286	}
1287
1288	/*
1289	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1290	* preemption hook code path acquires the current VMCS.
1291	*/
1292	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1293
1294	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1295	if (RT_SUCCESS(rc))
1296	{
1297	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1298
1299	/*
1300	* If we are switching to a VMCS that was executed on a different host CPU or was
1301	* never executed before, flag that we need to export the host state before executing
1302	* guest/nested-guest code using hardware-assisted VMX.
1303	*
1304	* This could probably be done in a preemptible context since the preemption hook
1305	* will flag the necessary change in host context. However, since preemption is
1306	* already disabled and to avoid making assumptions about host specific code in
1307	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1308	* disabled.
1309	*/
1310	if (pVmcsInfoTo->idHostCpu == RTMpCpuId())
1311	{ /* likely */ }
1312	else
1313	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1314
1315	ASMSetFlags(fEFlags);
1316
1317	/*
1318	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1319	* flag that we need to update the host MSR values there. Even if we decide in the
1320	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1321	* if its content differs, we would have to update the host MSRs anyway.
1322	*/
1323	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1324	}
1325	else
1326	ASMSetFlags(fEFlags);
1327	return rc;
1328	}
1329	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1330
1331
1332	/**
1333	* Updates the VM's last error record.
1334	*
1335	* If there was a VMX instruction error, reads the error data from the VMCS and
1336	* updates VCPU's last error record as well.
1337	*
1338	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1339	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1340	* VERR_VMX_INVALID_VMCS_FIELD.
1341	* @param rc The error code.
1342	*/
1343	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1344	{
1345	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1346	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1347	{
1348	AssertPtrReturnVoid(pVCpu);
1349	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1350	}
1351	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1352	}
1353
1354
1355	#ifdef VBOX_STRICT
1356	/**
1357	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1358	* transient structure.
1359	*
1360	* @returns VBox status code.
1361	* @param pVmxTransient The VMX-transient structure.
1362	*
1363	* @remarks No-long-jump zone!!!
1364	*/
1365	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1366	{
1367	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1368	AssertRCReturn(rc, rc);
1369	return VINF_SUCCESS;
1370	}
1371
1372
1373	/**
1374	* Reads the VM-entry exception error code field from the VMCS into
1375	* the VMX transient structure.
1376	*
1377	* @returns VBox status code.
1378	* @param pVmxTransient The VMX-transient structure.
1379	*
1380	* @remarks No-long-jump zone!!!
1381	*/
1382	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1383	{
1384	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1385	AssertRCReturn(rc, rc);
1386	return VINF_SUCCESS;
1387	}
1388
1389
1390	/**
1391	* Reads the VM-entry exception error code field from the VMCS into
1392	* the VMX transient structure.
1393	*
1394	* @returns VBox status code.
1395	* @param pVmxTransient The VMX-transient structure.
1396	*
1397	* @remarks No-long-jump zone!!!
1398	*/
1399	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1400	{
1401	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1402	AssertRCReturn(rc, rc);
1403	return VINF_SUCCESS;
1404	}
1405	#endif /* VBOX_STRICT */
1406
1407
1408	/**
1409	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1410	* transient structure.
1411	*
1412	* @returns VBox status code.
1413	* @param pVmxTransient The VMX-transient structure.
1414	*/
1415	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1416	{
1417	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1418	{
1419	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1420	AssertRCReturn(rc,rc);
1421	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1422	}
1423	return VINF_SUCCESS;
1424	}
1425
1426
1427	/**
1428	* Reads the VM-exit interruption error code from the VMCS into the VMX
1429	* transient structure.
1430	*
1431	* @returns VBox status code.
1432	* @param pVmxTransient The VMX-transient structure.
1433	*/
1434	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1435	{
1436	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1437	{
1438	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1439	AssertRCReturn(rc, rc);
1440	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1441	}
1442	return VINF_SUCCESS;
1443	}
1444
1445
1446	/**
1447	* Reads the VM-exit instruction length field from the VMCS into the VMX
1448	* transient structure.
1449	*
1450	* @returns VBox status code.
1451	* @param pVmxTransient The VMX-transient structure.
1452	*/
1453	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1454	{
1455	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1456	{
1457	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1458	AssertRCReturn(rc, rc);
1459	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1460	}
1461	return VINF_SUCCESS;
1462	}
1463
1464
1465	/**
1466	* Reads the VM-exit instruction-information field from the VMCS into
1467	* the VMX transient structure.
1468	*
1469	* @returns VBox status code.
1470	* @param pVmxTransient The VMX-transient structure.
1471	*/
1472	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1473	{
1474	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1475	{
1476	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1477	AssertRCReturn(rc, rc);
1478	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1479	}
1480	return VINF_SUCCESS;
1481	}
1482
1483
1484	/**
1485	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1486	*
1487	* @returns VBox status code.
1488	* @param pVCpu The cross context virtual CPU structure of the
1489	* calling EMT. (Required for the VMCS cache case.)
1490	* @param pVmxTransient The VMX-transient structure.
1491	*/
1492	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1493	{
1494	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1495	{
1496	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1497	AssertRCReturn(rc, rc);
1498	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1499	}
1500	return VINF_SUCCESS;
1501	}
1502
1503
1504	/**
1505	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1506	*
1507	* @returns VBox status code.
1508	* @param pVCpu The cross context virtual CPU structure of the
1509	* calling EMT. (Required for the VMCS cache case.)
1510	* @param pVmxTransient The VMX-transient structure.
1511	*/
1512	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1513	{
1514	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1515	{
1516	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1517	AssertRCReturn(rc, rc);
1518	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1519	}
1520	return VINF_SUCCESS;
1521	}
1522
1523
1524	/**
1525	* Reads the IDT-vectoring information field from the VMCS into the VMX
1526	* transient structure.
1527	*
1528	* @returns VBox status code.
1529	* @param pVmxTransient The VMX-transient structure.
1530	*
1531	* @remarks No-long-jump zone!!!
1532	*/
1533	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1534	{
1535	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1536	{
1537	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1538	AssertRCReturn(rc, rc);
1539	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1540	}
1541	return VINF_SUCCESS;
1542	}
1543
1544
1545	/**
1546	* Reads the IDT-vectoring error code from the VMCS into the VMX
1547	* transient structure.
1548	*
1549	* @returns VBox status code.
1550	* @param pVmxTransient The VMX-transient structure.
1551	*/
1552	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1553	{
1554	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1555	{
1556	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1557	AssertRCReturn(rc, rc);
1558	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1559	}
1560	return VINF_SUCCESS;
1561	}
1562
1563
1564	/**
1565	* Enters VMX root mode operation on the current CPU.
1566	*
1567	* @returns VBox status code.
1568	* @param pVM The cross context VM structure. Can be
1569	* NULL, after a resume.
1570	* @param HCPhysCpuPage Physical address of the VMXON region.
1571	* @param pvCpuPage Pointer to the VMXON region.
1572	*/
1573	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1574	{
1575	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1576	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1577	Assert(pvCpuPage);
1578	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1579
1580	if (pVM)
1581	{
1582	/* Write the VMCS revision identifier to the VMXON region. */
1583	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1584	}
1585
1586	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1587	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1588
1589	/* Enable the VMX bit in CR4 if necessary. */
1590	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1591
1592	/* Enter VMX root mode. */
1593	int rc = VMXEnable(HCPhysCpuPage);
1594	if (RT_FAILURE(rc))
1595	{
1596	if (!(uOldCr4 & X86_CR4_VMXE))
1597	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1598
1599	if (pVM)
1600	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1601	}
1602
1603	/* Restore interrupts. */
1604	ASMSetFlags(fEFlags);
1605	return rc;
1606	}
1607
1608
1609	/**
1610	* Exits VMX root mode operation on the current CPU.
1611	*
1612	* @returns VBox status code.
1613	*/
1614	static int hmR0VmxLeaveRootMode(void)
1615	{
1616	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1617
1618	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1619	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1620
1621	/* If we're for some reason not in VMX root mode, then don't leave it. */
1622	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1623
1624	int rc;
1625	if (uHostCr4 & X86_CR4_VMXE)
1626	{
1627	/* Exit VMX root mode and clear the VMX bit in CR4. */
1628	VMXDisable();
1629	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1630	rc = VINF_SUCCESS;
1631	}
1632	else
1633	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1634
1635	/* Restore interrupts. */
1636	ASMSetFlags(fEFlags);
1637	return rc;
1638	}
1639
1640
1641	/**
1642	* Allocates and maps a physically contiguous page. The allocated page is
1643	* zero'd out (used by various VT-x structures).
1644	*
1645	* @returns IPRT status code.
1646	* @param pMemObj Pointer to the ring-0 memory object.
1647	* @param ppVirt Where to store the virtual address of the allocation.
1648	* @param pHCPhys Where to store the physical address of the allocation.
1649	*/
1650	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1651	{
1652	AssertPtr(pMemObj);
1653	AssertPtr(ppVirt);
1654	AssertPtr(pHCPhys);
1655	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1656	if (RT_FAILURE(rc))
1657	return rc;
1658	ppVirt = RTR0MemObjAddress(pMemObj);
1659	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1660	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1661	return VINF_SUCCESS;
1662	}
1663
1664
1665	/**
1666	* Frees and unmaps an allocated, physical page.
1667	*
1668	* @param pMemObj Pointer to the ring-0 memory object.
1669	* @param ppVirt Where to re-initialize the virtual address of allocation as
1670	* 0.
1671	* @param pHCPhys Where to re-initialize the physical address of the
1672	* allocation as 0.
1673	*/
1674	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1675	{
1676	AssertPtr(pMemObj);
1677	AssertPtr(ppVirt);
1678	AssertPtr(pHCPhys);
1679	/* NULL is valid, accepted and ignored by the free function below. */
1680	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1681	*pMemObj = NIL_RTR0MEMOBJ;
1682	*ppVirt = NULL;
1683	*pHCPhys = NIL_RTHCPHYS;
1684	}
1685
1686
1687	/**
1688	* Initializes a VMCS info. object.
1689	*
1690	* @param pVmcsInfo The VMCS info. object.
1691	*/
1692	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1693	{
1694	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1695
1696	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1697	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1698	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1699	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1700	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1701	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1702	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1703	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1704	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1705	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1706	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1707	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1708	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1709	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1710	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1711	pVmcsInfo->idHostCpu = NIL_RTCPUID;
1712	}
1713
1714
1715	/**
1716	* Frees the VT-x structures for a VMCS info. object.
1717	*
1718	* @param pVM The cross context VM structure.
1719	* @param pVmcsInfo The VMCS info. object.
1720	*/
1721	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1722	{
1723	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1724
1725	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1726	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1727	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1728	#endif
1729
1730	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1731	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1732
1733	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1734	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1735	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1736
1737	hmR0VmxInitVmcsInfo(pVmcsInfo);
1738	}
1739
1740
1741	/**
1742	* Allocates the VT-x structures for a VMCS info. object.
1743	*
1744	* @returns VBox status code.
1745	* @param pVCpu The cross context virtual CPU structure.
1746	* @param pVmcsInfo The VMCS info. object.
1747	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1748	*/
1749	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1750	{
1751	PVM pVM = pVCpu->CTX_SUFF(pVM);
1752
1753	/* Allocate the guest VM control structure (VMCS). */
1754	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1755	if (RT_SUCCESS(rc))
1756	{
1757	if (!fIsNstGstVmcs)
1758	{
1759	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1760	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1761	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1762	#endif
1763	if (RT_SUCCESS(rc))
1764	{
1765	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1766	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1767	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1768	{
1769	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1770	NULL /* pR3Ptr /, NULL / pRCPtr */);
1771	}
1772	}
1773	}
1774	else
1775	{
1776	/* We don't yet support exposing VMCS shadowing to the guest. */
1777	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1778	Assert(!pVmcsInfo->pvShadowVmcs);
1779
1780	/* The host-physical address of the virtual-APIC page in guest memory is taken directly. */
1781	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1782	Assert(!pVmcsInfo->pbVirtApic);
1783	}
1784
1785	if (RT_SUCCESS(rc))
1786	{
1787	/*
1788	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1789	* transparent accesses of specific MSRs.
1790	*
1791	* If the condition for enabling MSR bitmaps changes here, don't forget to
1792	* update HMIsMsrBitmapActive().
1793	*
1794	* We don't share MSR bitmaps between the guest and nested-guest as we then
1795	* don't need to care about carefully restoring the guest MSR bitmap.
1796	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1797	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1798	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1799	* we do that later while merging VMCS.
1800	*/
1801	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1802	{
1803	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1804	if ( RT_SUCCESS(rc)
1805	&& !fIsNstGstVmcs)
1806	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1807	}
1808
1809	if (RT_SUCCESS(rc))
1810	{
1811	/*
1812	* Allocate the VM-entry MSR-load area for the guest MSRs.
1813	*
1814	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1815	* the guest and nested-guest.
1816	*/
1817	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1818	&pVmcsInfo->HCPhysGuestMsrLoad);
1819	if (RT_SUCCESS(rc))
1820	{
1821	/*
1822	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1823	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1824	*/
1825	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1826	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1827	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1828
1829	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1830	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1831	&pVmcsInfo->HCPhysHostMsrLoad);
1832	}
1833	}
1834	}
1835	}
1836
1837	return rc;
1838	}
1839
1840
1841	/**
1842	* Free all VT-x structures for the VM.
1843	*
1844	* @returns IPRT status code.
1845	* @param pVM The cross context VM structure.
1846	*/
1847	static void hmR0VmxStructsFree(PVM pVM)
1848	{
1849	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1850	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1851	#endif
1852	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1853
1854	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1855	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1856	{
1857	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1858	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1859	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1860	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1861	}
1862	#endif
1863
1864	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1865	{
1866	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1867	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1868	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1869	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1870	if (pVM->cpum.ro.GuestFeatures.fVmx)
1871	{
1872	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1873	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1874	}
1875	#endif
1876	}
1877	}
1878
1879
1880	/**
1881	* Allocate all VT-x structures for the VM.
1882	*
1883	* @returns IPRT status code.
1884	* @param pVM The cross context VM structure.
1885	*/
1886	static int hmR0VmxStructsAlloc(PVM pVM)
1887	{
1888	/*
1889	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1890	* The VMCS size cannot be more than 4096 bytes.
1891	*
1892	* See Intel spec. Appendix A.1 "Basic VMX Information".
1893	*/
1894	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1895	if (cbVmcs <= X86_PAGE_4K_SIZE)
1896	{ /* likely */ }
1897	else
1898	{
1899	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1900	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1901	}
1902
1903	/*
1904	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1905	*/
1906	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1907	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1908	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1909	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1910	#endif
1911
1912	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1913	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1914	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1915
1916	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1917	{
1918	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1919	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1920	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1921	}
1922
1923	/*
1924	* Allocate per-VM VT-x structures.
1925	*/
1926	int rc = VINF_SUCCESS;
1927	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1928	/* Allocate crash-dump magic scratch page. */
1929	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1930	if (RT_FAILURE(rc))
1931	{
1932	hmR0VmxStructsFree(pVM);
1933	return rc;
1934	}
1935	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1936	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1937	#endif
1938
1939	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1940	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1941	{
1942	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1943	&pVM->hm.s.vmx.HCPhysApicAccess);
1944	if (RT_FAILURE(rc))
1945	{
1946	hmR0VmxStructsFree(pVM);
1947	return rc;
1948	}
1949	}
1950
1951	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1952	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1953	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1954	{
1955	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1956	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1957	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1958	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1959	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1960	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1961	{
1962	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1963	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1964	if (RT_SUCCESS(rc))
1965	{
1966	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1967	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1968	}
1969	}
1970	else
1971	rc = VERR_NO_MEMORY;
1972
1973	if (RT_FAILURE(rc))
1974	{
1975	hmR0VmxStructsFree(pVM);
1976	return rc;
1977	}
1978	}
1979	#endif
1980
1981	/*
1982	* Initialize per-VCPU VT-x structures.
1983	*/
1984	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1985	{
1986	/* Allocate the guest VMCS structures. */
1987	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1988	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1989	if (RT_SUCCESS(rc))
1990	{
1991	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1992	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1993	if (pVM->cpum.ro.GuestFeatures.fVmx)
1994	{
1995	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1996	if (RT_SUCCESS(rc))
1997	{ /* likely */ }
1998	else
1999	break;
2000	}
2001	#endif
2002	}
2003	else
2004	break;
2005	}
2006
2007	if (RT_FAILURE(rc))
2008	{
2009	hmR0VmxStructsFree(pVM);
2010	return rc;
2011	}
2012
2013	return VINF_SUCCESS;
2014	}
2015
2016	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2017	/**
2018	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2019	*
2020	* @returns @c true if the MSR is intercepted, @c false otherwise.
2021	* @param pvMsrBitmap The MSR bitmap.
2022	* @param offMsr The MSR byte offset.
2023	* @param iBit The bit offset from the byte offset.
2024	*/
2025	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2026	{
2027	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2028	Assert(pbMsrBitmap);
2029	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2030	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2031	}
2032	#endif
2033
2034	/**
2035	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2036	*
2037	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2038	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2039	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2040	* the read/write access of this MSR.
2041	*
2042	* @param pVCpu The cross context virtual CPU structure.
2043	* @param pVmcsInfo The VMCS info. object.
2044	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2045	* @param idMsr The MSR value.
2046	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2047	* include both a read -and- a write permission!
2048	*
2049	* @sa CPUMGetVmxMsrPermission.
2050	* @remarks Can be called with interrupts disabled.
2051	*/
2052	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2053	{
2054	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2055	Assert(pbMsrBitmap);
2056	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2057
2058	/*
2059	* MSR-bitmap Layout:
2060	* Byte index MSR range Interpreted as
2061	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2062	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2063	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2064	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2065	*
2066	* A bit corresponding to an MSR within the above range causes a VM-exit
2067	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2068	* the MSR range, it always cause a VM-exit.
2069	*
2070	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2071	*/
2072	uint16_t const offBitmapRead = 0;
2073	uint16_t const offBitmapWrite = 0x800;
2074	uint16_t offMsr;
2075	int32_t iBit;
2076	if (idMsr <= UINT32_C(0x00001fff))
2077	{
2078	offMsr = 0;
2079	iBit = idMsr;
2080	}
2081	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2082	{
2083	offMsr = 0x400;
2084	iBit = idMsr - UINT32_C(0xc0000000);
2085	}
2086	else
2087	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2088
2089	/*
2090	* Set the MSR read permission.
2091	*/
2092	uint16_t const offMsrRead = offBitmapRead + offMsr;
2093	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2094	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2095	{
2096	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2097	bool const fClear = !fIsNstGstVmcs ? true
2098	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2099	#else
2100	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2101	bool const fClear = true;
2102	#endif
2103	if (fClear)
2104	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2105	}
2106	else
2107	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2108
2109	/*
2110	* Set the MSR write permission.
2111	*/
2112	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2113	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2114	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2115	{
2116	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2117	bool const fClear = !fIsNstGstVmcs ? true
2118	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2119	#else
2120	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2121	bool const fClear = true;
2122	#endif
2123	if (fClear)
2124	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2125	}
2126	else
2127	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2128	}
2129
2130
2131	/**
2132	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2133	* area.
2134	*
2135	* @returns VBox status code.
2136	* @param pVCpu The cross context virtual CPU structure.
2137	* @param pVmcsInfo The VMCS info. object.
2138	* @param cMsrs The number of MSRs.
2139	*/
2140	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2141	{
2142	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2143	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2144	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2145	{
2146	/* Commit the MSR counts to the VMCS and update the cache. */
2147	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2148	{
2149	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2150	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2151	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2152	AssertRCReturn(rc, rc);
2153
2154	pVmcsInfo->cEntryMsrLoad = cMsrs;
2155	pVmcsInfo->cExitMsrStore = cMsrs;
2156	pVmcsInfo->cExitMsrLoad = cMsrs;
2157	}
2158	return VINF_SUCCESS;
2159	}
2160
2161	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2162	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2163	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2164	}
2165
2166
2167	/**
2168	* Adds a new (or updates the value of an existing) guest/host MSR
2169	* pair to be swapped during the world-switch as part of the
2170	* auto-load/store MSR area in the VMCS.
2171	*
2172	* @returns VBox status code.
2173	* @param pVCpu The cross context virtual CPU structure.
2174	* @param pVmxTransient The VMX-transient structure.
2175	* @param idMsr The MSR.
2176	* @param uGuestMsrValue Value of the guest MSR.
2177	* @param fSetReadWrite Whether to set the guest read/write access of this
2178	* MSR (thus not causing a VM-exit).
2179	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2180	* necessary.
2181	*/
2182	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2183	bool fSetReadWrite, bool fUpdateHostMsr)
2184	{
2185	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2186	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2187	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2188	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2189	uint32_t i;
2190
2191	/* Paranoia. */
2192	Assert(pGuestMsrLoad);
2193
2194	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2195
2196	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2197	for (i = 0; i < cMsrs; i++)
2198	{
2199	if (pGuestMsrLoad[i].u32Msr == idMsr)
2200	break;
2201	}
2202
2203	bool fAdded = false;
2204	if (i == cMsrs)
2205	{
2206	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2207	++cMsrs;
2208	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2209	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2210
2211	/* Set the guest to read/write this MSR without causing VM-exits. */
2212	if ( fSetReadWrite
2213	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2214	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2215
2216	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
2217	fAdded = true;
2218	}
2219
2220	/* Update the MSR value for the newly added or already existing MSR. */
2221	pGuestMsrLoad[i].u32Msr = idMsr;
2222	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2223
2224	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2225	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2226	{
2227	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2228	pGuestMsrStore[i].u32Msr = idMsr;
2229	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2230	}
2231
2232	/* Update the corresponding slot in the host MSR area. */
2233	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2234	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2235	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2236	pHostMsr[i].u32Msr = idMsr;
2237
2238	/*
2239	* Only if the caller requests to update the host MSR value AND we've newly added the
2240	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2241	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2242	*
2243	* We do this for performance reasons since reading MSRs may be quite expensive.
2244	*/
2245	if (fAdded)
2246	{
2247	if (fUpdateHostMsr)
2248	{
2249	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2250	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2251	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2252	}
2253	else
2254	{
2255	/* Someone else can do the work. */
2256	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2257	}
2258	}
2259	return VINF_SUCCESS;
2260	}
2261
2262
2263	/**
2264	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2265	* auto-load/store MSR area in the VMCS.
2266	*
2267	* @returns VBox status code.
2268	* @param pVCpu The cross context virtual CPU structure.
2269	* @param pVmxTransient The VMX-transient structure.
2270	* @param idMsr The MSR.
2271	*/
2272	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2273	{
2274	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2275	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2276	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2277	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2278
2279	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2280
2281	for (uint32_t i = 0; i < cMsrs; i++)
2282	{
2283	/* Find the MSR. */
2284	if (pGuestMsrLoad[i].u32Msr == idMsr)
2285	{
2286	/*
2287	* If it's the last MSR, we only need to reduce the MSR count.
2288	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2289	*/
2290	if (i < cMsrs - 1)
2291	{
2292	/* Remove it from the VM-entry MSR-load area. */
2293	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2294	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2295
2296	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2297	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2298	{
2299	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2300	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2301	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2302	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2303	}
2304
2305	/* Remove it from the VM-exit MSR-load area. */
2306	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2307	Assert(pHostMsr[i].u32Msr == idMsr);
2308	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2309	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2310	}
2311
2312	/* Reduce the count to reflect the removed MSR and bail. */
2313	--cMsrs;
2314	break;
2315	}
2316	}
2317
2318	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2319	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2320	{
2321	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2322	AssertRCReturn(rc, rc);
2323
2324	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2325	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2326	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2327
2328	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2329	return VINF_SUCCESS;
2330	}
2331
2332	return VERR_NOT_FOUND;
2333	}
2334
2335
2336	/**
2337	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2338	*
2339	* @returns @c true if found, @c false otherwise.
2340	* @param pVmcsInfo The VMCS info. object.
2341	* @param idMsr The MSR to find.
2342	*/
2343	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2344	{
2345	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2346	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2347	Assert(pMsrs);
2348	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2349	for (uint32_t i = 0; i < cMsrs; i++)
2350	{
2351	if (pMsrs[i].u32Msr == idMsr)
2352	return true;
2353	}
2354	return false;
2355	}
2356
2357
2358	/**
2359	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2360	*
2361	* @param pVCpu The cross context virtual CPU structure.
2362	* @param pVmcsInfo The VMCS info. object.
2363	*
2364	* @remarks No-long-jump zone!!!
2365	*/
2366	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2367	{
2368	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2369
2370	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2371	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2372	Assert(pHostMsrLoad);
2373	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2374	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2375	for (uint32_t i = 0; i < cMsrs; i++)
2376	{
2377	/*
2378	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2379	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2380	*/
2381	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2382	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2383	else
2384	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2385	}
2386	}
2387
2388
2389	/**
2390	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2391	* perform lazy restoration of the host MSRs while leaving VT-x.
2392	*
2393	* @param pVCpu The cross context virtual CPU structure.
2394	*
2395	* @remarks No-long-jump zone!!!
2396	*/
2397	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2398	{
2399	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2400
2401	/*
2402	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2403	*/
2404	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2405	{
2406	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2407	#if HC_ARCH_BITS == 64
2408	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2409	{
2410	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2411	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2412	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2413	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2414	}
2415	#endif
2416	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2417	}
2418	}
2419
2420
2421	/**
2422	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2423	* lazily while leaving VT-x.
2424	*
2425	* @returns true if it does, false otherwise.
2426	* @param pVCpu The cross context virtual CPU structure.
2427	* @param idMsr The MSR to check.
2428	*/
2429	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2430	{
2431	NOREF(pVCpu);
2432	#if HC_ARCH_BITS == 64
2433	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2434	{
2435	switch (idMsr)
2436	{
2437	case MSR_K8_LSTAR:
2438	case MSR_K6_STAR:
2439	case MSR_K8_SF_MASK:
2440	case MSR_K8_KERNEL_GS_BASE:
2441	return true;
2442	}
2443	}
2444	#else
2445	RT_NOREF(pVCpu, idMsr);
2446	#endif
2447	return false;
2448	}
2449
2450
2451	/**
2452	* Loads a set of guests MSRs to allow read/passthru to the guest.
2453	*
2454	* The name of this function is slightly confusing. This function does NOT
2455	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2456	* common prefix for functions dealing with "lazy restoration" of the shared
2457	* MSRs.
2458	*
2459	* @param pVCpu The cross context virtual CPU structure.
2460	*
2461	* @remarks No-long-jump zone!!!
2462	*/
2463	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2464	{
2465	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2466	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2467
2468	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2469	#if HC_ARCH_BITS == 64
2470	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2471	{
2472	/*
2473	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2474	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2475	* we can skip a few MSR writes.
2476	*
2477	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2478	* guest MSR values in the guest-CPU context might be different to what's currently
2479	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2480	* CPU, see @bugref{8728}.
2481	*/
2482	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2483	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2484	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2485	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2486	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2487	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2488	{
2489	#ifdef VBOX_STRICT
2490	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2491	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2492	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2493	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2494	#endif
2495	}
2496	else
2497	{
2498	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2499	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2500	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2501	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2502	}
2503	}
2504	#endif
2505	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2506	}
2507
2508
2509	/**
2510	* Performs lazy restoration of the set of host MSRs if they were previously
2511	* loaded with guest MSR values.
2512	*
2513	* @param pVCpu The cross context virtual CPU structure.
2514	*
2515	* @remarks No-long-jump zone!!!
2516	* @remarks The guest MSRs should have been saved back into the guest-CPU
2517	* context by hmR0VmxImportGuestState()!!!
2518	*/
2519	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2520	{
2521	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2522	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2523
2524	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2525	{
2526	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2527	#if HC_ARCH_BITS == 64
2528	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2529	{
2530	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2531	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2532	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2533	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2534	}
2535	#endif
2536	}
2537	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2538	}
2539
2540
2541	/**
2542	* Verifies that our cached values of the VMCS fields are all consistent with
2543	* what's actually present in the VMCS.
2544	*
2545	* @returns VBox status code.
2546	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2547	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2548	* VMCS content. HMCPU error-field is
2549	* updated, see VMX_VCI_XXX.
2550	* @param pVCpu The cross context virtual CPU structure.
2551	* @param pVmcsInfo The VMCS info. object.
2552	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2553	*/
2554	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2555	{
2556	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2557
2558	uint32_t u32Val;
2559	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2560	AssertRCReturn(rc, rc);
2561	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2562	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2563	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2564	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2565
2566	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2567	AssertRCReturn(rc, rc);
2568	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2569	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2570	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2571	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2572
2573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2574	AssertRCReturn(rc, rc);
2575	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2576	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2577	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2578	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2579
2580	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2581	AssertRCReturn(rc, rc);
2582	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2583	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2584	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2585	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2586
2587	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2588	{
2589	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2590	AssertRCReturn(rc, rc);
2591	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2592	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2593	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2594	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2595	}
2596
2597	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2598	AssertRCReturn(rc, rc);
2599	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2600	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2601	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2602	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2603
2604	uint64_t u64Val;
2605	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2606	AssertRCReturn(rc, rc);
2607	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2608	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2609	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2610	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2611
2612	NOREF(pcszVmcs);
2613	return VINF_SUCCESS;
2614	}
2615
2616
2617	#ifdef VBOX_STRICT
2618	/**
2619	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2620	*
2621	* @param pVCpu The cross context virtual CPU structure.
2622	* @param pVmcsInfo The VMCS info. object.
2623	*/
2624	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2625	{
2626	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2627
2628	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2629	{
2630	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2631	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2632	uint64_t uVmcsEferMsrVmcs;
2633	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2634	AssertRC(rc);
2635
2636	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2637	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2638	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2639	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2640	}
2641	}
2642
2643
2644	/**
2645	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2646	* VMCS are correct.
2647	*
2648	* @param pVCpu The cross context virtual CPU structure.
2649	* @param pVmcsInfo The VMCS info. object.
2650	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2651	*/
2652	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2653	{
2654	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2655
2656	/* Read the various MSR-area counts from the VMCS. */
2657	uint32_t cEntryLoadMsrs;
2658	uint32_t cExitStoreMsrs;
2659	uint32_t cExitLoadMsrs;
2660	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2661	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2663
2664	/* Verify all the MSR counts are the same. */
2665	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2666	Assert(cExitStoreMsrs == cExitLoadMsrs);
2667	uint32_t const cMsrs = cExitLoadMsrs;
2668
2669	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2670	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2671
2672	/* Verify the MSR counts are within the allocated page size. */
2673	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2674
2675	/* Verify the relevant contents of the MSR areas match. */
2676	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2677	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2678	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2679	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2680	for (uint32_t i = 0; i < cMsrs; i++)
2681	{
2682	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2683	if (fSeparateExitMsrStorePage)
2684	{
2685	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2686	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2687	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2688	}
2689
2690	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2691	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2692	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2693
2694	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2695	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2696	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2697	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2698
2699	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2700	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2701	if (fIsEferMsr)
2702	{
2703	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2704	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2705	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2706	}
2707
2708	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2709	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2710	{
2711	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2712	if (fIsEferMsr)
2713	{
2714	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2715	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2716	}
2717	else
2718	{
2719	if (!fIsNstGstVmcs)
2720	{
2721	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2722	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2723	}
2724	else
2725	{
2726	/*
2727	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2728	* execute a nested-guest with MSR passthrough.
2729	*
2730	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2731	* allow passthrough too.
2732	*/
2733	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2734	Assert(pvMsrBitmapNstGst);
2735	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2736	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2737	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2738	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2739	}
2740	}
2741	}
2742
2743	/* Move to the next MSR. */
2744	pHostMsrLoad++;
2745	pGuestMsrLoad++;
2746	pGuestMsrStore++;
2747	}
2748	}
2749	#endif /* VBOX_STRICT */
2750
2751
2752	/**
2753	* Flushes the TLB using EPT.
2754	*
2755	* @returns VBox status code.
2756	* @param pVCpu The cross context virtual CPU structure of the calling
2757	* EMT. Can be NULL depending on @a enmTlbFlush.
2758	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2759	* enmTlbFlush.
2760	* @param enmTlbFlush Type of flush.
2761	*
2762	* @remarks Caller is responsible for making sure this function is called only
2763	* when NestedPaging is supported and providing @a enmTlbFlush that is
2764	* supported by the CPU.
2765	* @remarks Can be called with interrupts disabled.
2766	*/
2767	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2768	{
2769	uint64_t au64Descriptor[2];
2770	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2771	au64Descriptor[0] = 0;
2772	else
2773	{
2774	Assert(pVCpu);
2775	Assert(pVmcsInfo);
2776	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2777	}
2778	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2779
2780	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2781	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2782
2783	if ( RT_SUCCESS(rc)
2784	&& pVCpu)
2785	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2786	}
2787
2788
2789	/**
2790	* Flushes the TLB using VPID.
2791	*
2792	* @returns VBox status code.
2793	* @param pVCpu The cross context virtual CPU structure of the calling
2794	* EMT. Can be NULL depending on @a enmTlbFlush.
2795	* @param enmTlbFlush Type of flush.
2796	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2797	* on @a enmTlbFlush).
2798	*
2799	* @remarks Can be called with interrupts disabled.
2800	*/
2801	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2802	{
2803	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2804
2805	uint64_t au64Descriptor[2];
2806	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2807	{
2808	au64Descriptor[0] = 0;
2809	au64Descriptor[1] = 0;
2810	}
2811	else
2812	{
2813	AssertPtr(pVCpu);
2814	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2815	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2816	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2817	au64Descriptor[1] = GCPtr;
2818	}
2819
2820	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2821	AssertMsg(rc == VINF_SUCCESS,
2822	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2823
2824	if ( RT_SUCCESS(rc)
2825	&& pVCpu)
2826	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2827	NOREF(rc);
2828	}
2829
2830
2831	/**
2832	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2833	* otherwise there is nothing really to invalidate.
2834	*
2835	* @returns VBox status code.
2836	* @param pVCpu The cross context virtual CPU structure.
2837	* @param GCVirt Guest virtual address of the page to invalidate.
2838	*/
2839	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2840	{
2841	AssertPtr(pVCpu);
2842	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2843
2844	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2845	{
2846	/*
2847	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2848	* the EPT case. See @bugref{6043} and @bugref{6177}.
2849	*
2850	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2851	* as this function maybe called in a loop with individual addresses.
2852	*/
2853	PVM pVM = pVCpu->CTX_SUFF(pVM);
2854	if (pVM->hm.s.vmx.fVpid)
2855	{
2856	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2857
2858	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2859	/*
2860	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2861	* where executing INVVPID outside 64-bit mode does not flush translations of
2862	* 64-bit linear addresses, see @bugref{6208#c72}.
2863	*/
2864	if (RT_HI_U32(GCVirt))
2865	fVpidFlush = false;
2866	#endif
2867
2868	if (fVpidFlush)
2869	{
2870	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2871	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2872	}
2873	else
2874	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2875	}
2876	else if (pVM->hm.s.fNestedPaging)
2877	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2878	}
2879
2880	return VINF_SUCCESS;
2881	}
2882
2883
2884	/**
2885	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2886	* case where neither EPT nor VPID is supported by the CPU.
2887	*
2888	* @param pHostCpu The HM physical-CPU structure.
2889	* @param pVCpu The cross context virtual CPU structure.
2890	*
2891	* @remarks Called with interrupts disabled.
2892	*/
2893	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2894	{
2895	AssertPtr(pVCpu);
2896	AssertPtr(pHostCpu);
2897
2898	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2899
2900	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2901	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2902	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2903	pVCpu->hm.s.fForceTLBFlush = false;
2904	return;
2905	}
2906
2907
2908	/**
2909	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2910	*
2911	* @param pHostCpu The HM physical-CPU structure.
2912	* @param pVCpu The cross context virtual CPU structure.
2913	* @param pVmcsInfo The VMCS info. object.
2914	*
2915	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2916	* nomenclature. The reason is, to avoid confusion in compare statements
2917	* since the host-CPU copies are named "ASID".
2918	*
2919	* @remarks Called with interrupts disabled.
2920	*/
2921	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2922	{
2923	#ifdef VBOX_WITH_STATISTICS
2924	bool fTlbFlushed = false;
2925	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2926	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2927	if (!fTlbFlushed) \
2928	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2929	} while (0)
2930	#else
2931	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2932	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2933	#endif
2934
2935	AssertPtr(pVCpu);
2936	AssertPtr(pHostCpu);
2937	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2938
2939	PVM pVM = pVCpu->CTX_SUFF(pVM);
2940	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2941	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2942	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2943
2944	/*
2945	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2946	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2947	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2948	* cannot reuse the current ASID anymore.
2949	*/
2950	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2951	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2952	{
2953	++pHostCpu->uCurrentAsid;
2954	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2955	{
2956	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2957	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2958	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2959	}
2960
2961	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2962	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2963	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2964
2965	/*
2966	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2967	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2968	*/
2969	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2970	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2971	HMVMX_SET_TAGGED_TLB_FLUSHED();
2972	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2973	}
2974	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2975	{
2976	/*
2977	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2978	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2979	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2980	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2981	* mappings, see @bugref{6568}.
2982	*
2983	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2984	*/
2985	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2986	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2987	HMVMX_SET_TAGGED_TLB_FLUSHED();
2988	}
2989
2990	pVCpu->hm.s.fForceTLBFlush = false;
2991	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2992
2993	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2994	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2995	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2996	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2997	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2998	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2999	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3000	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3001	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3002
3003	/* Update VMCS with the VPID. */
3004	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3005	AssertRC(rc);
3006
3007	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3008	}
3009
3010
3011	/**
3012	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3013	*
3014	* @param pHostCpu The HM physical-CPU structure.
3015	* @param pVCpu The cross context virtual CPU structure.
3016	* @param pVmcsInfo The VMCS info. object.
3017	*
3018	* @remarks Called with interrupts disabled.
3019	*/
3020	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3021	{
3022	AssertPtr(pVCpu);
3023	AssertPtr(pHostCpu);
3024	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3025	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3026	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3027
3028	/*
3029	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3030	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3031	*/
3032	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3033	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3034	{
3035	pVCpu->hm.s.fForceTLBFlush = true;
3036	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3037	}
3038
3039	/* Check for explicit TLB flushes. */
3040	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3041	{
3042	pVCpu->hm.s.fForceTLBFlush = true;
3043	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3044	}
3045
3046	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3047	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3048
3049	if (pVCpu->hm.s.fForceTLBFlush)
3050	{
3051	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3052	pVCpu->hm.s.fForceTLBFlush = false;
3053	}
3054	}
3055
3056
3057	/**
3058	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3059	*
3060	* @param pHostCpu The HM physical-CPU structure.
3061	* @param pVCpu The cross context virtual CPU structure.
3062	*
3063	* @remarks Called with interrupts disabled.
3064	*/
3065	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3066	{
3067	AssertPtr(pVCpu);
3068	AssertPtr(pHostCpu);
3069	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3070	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3071	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3072
3073	/*
3074	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3075	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3076	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3077	* cannot reuse the current ASID anymore.
3078	*/
3079	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3080	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3081	{
3082	pVCpu->hm.s.fForceTLBFlush = true;
3083	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3084	}
3085
3086	/* Check for explicit TLB flushes. */
3087	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3088	{
3089	/*
3090	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3091	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3092	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3093	* include fExplicitFlush's too) - an obscure corner case.
3094	*/
3095	pVCpu->hm.s.fForceTLBFlush = true;
3096	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3097	}
3098
3099	PVM pVM = pVCpu->CTX_SUFF(pVM);
3100	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3101	if (pVCpu->hm.s.fForceTLBFlush)
3102	{
3103	++pHostCpu->uCurrentAsid;
3104	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3105	{
3106	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3107	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3108	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3109	}
3110
3111	pVCpu->hm.s.fForceTLBFlush = false;
3112	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3113	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3114	if (pHostCpu->fFlushAsidBeforeUse)
3115	{
3116	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3117	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3118	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3119	{
3120	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3121	pHostCpu->fFlushAsidBeforeUse = false;
3122	}
3123	else
3124	{
3125	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3126	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3127	}
3128	}
3129	}
3130
3131	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3132	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3133	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3134	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3135	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3136	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3137	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3138
3139	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3140	AssertRC(rc);
3141	}
3142
3143
3144	/**
3145	* Flushes the guest TLB entry based on CPU capabilities.
3146	*
3147	* @param pHostCpu The HM physical-CPU structure.
3148	* @param pVCpu The cross context virtual CPU structure.
3149	* @param pVmcsInfo The VMCS info. object.
3150	*
3151	* @remarks Called with interrupts disabled.
3152	*/
3153	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3154	{
3155	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3156	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3157	#endif
3158	PVM pVM = pVCpu->CTX_SUFF(pVM);
3159	switch (pVM->hm.s.vmx.enmTlbFlushType)
3160	{
3161	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3162	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3163	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3164	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3165	default:
3166	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3167	break;
3168	}
3169	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3170	}
3171
3172
3173	/**
3174	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3175	* TLB entries from the host TLB before VM-entry.
3176	*
3177	* @returns VBox status code.
3178	* @param pVM The cross context VM structure.
3179	*/
3180	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3181	{
3182	/*
3183	* Determine optimal flush type for nested paging.
3184	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3185	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3186	*/
3187	if (pVM->hm.s.fNestedPaging)
3188	{
3189	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3190	{
3191	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3192	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3193	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3194	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3195	else
3196	{
3197	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3198	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3199	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3200	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3201	}
3202
3203	/* Make sure the write-back cacheable memory type for EPT is supported. */
3204	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3205	{
3206	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3207	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3208	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3209	}
3210
3211	/* EPT requires a page-walk length of 4. */
3212	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3213	{
3214	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3215	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3217	}
3218	}
3219	else
3220	{
3221	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3222	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3223	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3224	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3225	}
3226	}
3227
3228	/*
3229	* Determine optimal flush type for VPID.
3230	*/
3231	if (pVM->hm.s.vmx.fVpid)
3232	{
3233	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3234	{
3235	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3236	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3237	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3238	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3239	else
3240	{
3241	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3242	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3243	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3244	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3245	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3246	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3247	pVM->hm.s.vmx.fVpid = false;
3248	}
3249	}
3250	else
3251	{
3252	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3253	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3254	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3255	pVM->hm.s.vmx.fVpid = false;
3256	}
3257	}
3258
3259	/*
3260	* Setup the handler for flushing tagged-TLBs.
3261	*/
3262	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3263	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3264	else if (pVM->hm.s.fNestedPaging)
3265	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3266	else if (pVM->hm.s.vmx.fVpid)
3267	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3268	else
3269	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3270	return VINF_SUCCESS;
3271	}
3272
3273
3274	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3275	/**
3276	* Sets up the shadow VMCS fields arrays.
3277	*
3278	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3279	* executing the guest.
3280	*
3281	* @returns VBox status code.
3282	* @param pVM The cross context VM structure.
3283	*/
3284	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3285	{
3286	/*
3287	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3288	* when the host does not support it.
3289	*/
3290	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3291	if ( !fGstVmwriteAll
3292	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3293	{ /* likely. */ }
3294	else
3295	{
3296	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3297	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3298	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3299	}
3300
3301	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3302	uint32_t cRwFields = 0;
3303	uint32_t cRoFields = 0;
3304	for (uint32_t i = 0; i < cVmcsFields; i++)
3305	{
3306	VMXVMCSFIELD VmcsField;
3307	VmcsField.u = g_aVmcsFields[i];
3308
3309	/*
3310	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3311	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3312	* in the shadow VMCS fields array as they would be redundant.
3313	*
3314	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3315	* we must not include it in the shadow VMCS fields array. Guests attempting to
3316	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3317	* the required behavior.
3318	*/
3319	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3320	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3321	{
3322	/*
3323	* Read-only fields are placed in a separate array so that while syncing shadow
3324	* VMCS fields later (which is more performance critical) we can avoid branches.
3325	*
3326	* However, if the guest can write to all fields (including read-only fields),
3327	* we treat it a as read/write field. Otherwise, writing to these fields would
3328	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3329	*/
3330	if ( fGstVmwriteAll
3331	\|\| !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3332	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3333	else
3334	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3335	}
3336	}
3337
3338	/* Update the counts. */
3339	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3340	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3341	return VINF_SUCCESS;
3342	}
3343
3344
3345	/**
3346	* Sets up the VMREAD and VMWRITE bitmaps.
3347	*
3348	* @param pVM The cross context VM structure.
3349	*/
3350	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3351	{
3352	/*
3353	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3354	*/
3355	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3356	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3357	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3358	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3359	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3360
3361	/*
3362	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3363	* VMREAD and VMWRITE bitmaps.
3364	*/
3365	{
3366	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3367	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3368	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3369	{
3370	uint32_t const uVmcsField = paShadowVmcsFields[i];
3371	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3372	Assert(uVmcsField >> 3 < cbBitmap);
3373	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3374	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3375	}
3376	}
3377
3378	/*
3379	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3380	* if the host supports VMWRITE to all supported VMCS fields.
3381	*/
3382	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3383	{
3384	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3385	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3386	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3387	{
3388	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3389	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3390	Assert(uVmcsField >> 3 < cbBitmap);
3391	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3392	}
3393	}
3394	}
3395	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3396
3397
3398	/**
3399	* Sets up the virtual-APIC page address for the VMCS.
3400	*
3401	* @returns VBox status code.
3402	* @param pVCpu The cross context virtual CPU structure.
3403	* @param pVmcsInfo The VMCS info. object.
3404	*/
3405	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3406	{
3407	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3408	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3409	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3410	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3411	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3412	}
3413
3414
3415	/**
3416	* Sets up the MSR-bitmap address for the VMCS.
3417	*
3418	* @returns VBox status code.
3419	* @param pVCpu The cross context virtual CPU structure.
3420	* @param pVmcsInfo The VMCS info. object.
3421	*/
3422	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3423	{
3424	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3425	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3426	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3427	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3428	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3429	}
3430
3431
3432	/**
3433	* Sets up the APIC-access page address for the VMCS.
3434	*
3435	* @returns VBox status code.
3436	* @param pVCpu The cross context virtual CPU structure.
3437	*/
3438	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3439	{
3440	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3441	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3442	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3443	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3444	}
3445
3446
3447	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3448	/**
3449	* Sets up the VMREAD bitmap address for the VMCS.
3450	*
3451	* @returns VBox status code.
3452	* @param pVCpu The cross context virtual CPU structure.
3453	*/
3454	DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3455	{
3456	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3457	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3458	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3459	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3460	}
3461
3462
3463	/**
3464	* Sets up the VMWRITE bitmap address for the VMCS.
3465	*
3466	* @returns VBox status code.
3467	* @param pVCpu The cross context virtual CPU structure.
3468	*/
3469	DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3470	{
3471	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3472	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3473	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3474	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3475	}
3476	#endif
3477
3478
3479	/**
3480	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3481	* in the VMCS.
3482	*
3483	* @returns VBox status code.
3484	* @param pVCpu The cross context virtual CPU structure.
3485	* @param pVmcsInfo The VMCS info. object.
3486	*/
3487	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3488	{
3489	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3490
3491	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3492	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3493	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3494
3495	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3496	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3497	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3498
3499	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3500	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3501	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3502
3503	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3504	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3505	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3506	AssertRCReturn(rc, rc);
3507	return VINF_SUCCESS;
3508	}
3509
3510
3511	/**
3512	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3513	*
3514	* @param pVCpu The cross context virtual CPU structure.
3515	* @param pVmcsInfo The VMCS info. object.
3516	*/
3517	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3518	{
3519	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3520
3521	/*
3522	* The guest can access the following MSRs (read, write) without causing
3523	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3524	*/
3525	PVM pVM = pVCpu->CTX_SUFF(pVM);
3526	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3527	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3528	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3529	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3530	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3531
3532	/*
3533	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3534	* associated with then. We never need to intercept access (writes need to be
3535	* executed without causing a VM-exit, reads will #GP fault anyway).
3536	*
3537	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3538	* read/write them. We swap the the guest/host MSR value using the
3539	* auto-load/store MSR area.
3540	*/
3541	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3542	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3543	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3544	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3545	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3546	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3547
3548	#if HC_ARCH_BITS == 64
3549	/*
3550	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3551	* required for 64-bit guests.
3552	*/
3553	if (pVM->hm.s.fAllow64BitGuests)
3554	{
3555	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3556	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3557	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3558	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3559	}
3560	#endif
3561
3562	/*
3563	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3564	*/
3565	#ifdef VBOX_STRICT
3566	Assert(pVmcsInfo->pvMsrBitmap);
3567	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3568	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3569	#endif
3570	}
3571
3572
3573	/**
3574	* Sets up pin-based VM-execution controls in the VMCS.
3575	*
3576	* @returns VBox status code.
3577	* @param pVCpu The cross context virtual CPU structure.
3578	* @param pVmcsInfo The VMCS info. object.
3579	*/
3580	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3581	{
3582	PVM pVM = pVCpu->CTX_SUFF(pVM);
3583	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3584	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3585
3586	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3587	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3588
3589	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3590	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3591
3592	/* Enable the VMX-preemption timer. */
3593	if (pVM->hm.s.vmx.fUsePreemptTimer)
3594	{
3595	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3596	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3597	}
3598
3599	#if 0
3600	/* Enable posted-interrupt processing. */
3601	if (pVM->hm.s.fPostedIntrs)
3602	{
3603	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3604	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3605	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3606	}
3607	#endif
3608
3609	if ((fVal & fZap) != fVal)
3610	{
3611	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3612	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3613	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3614	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3615	}
3616
3617	/* Commit it to the VMCS and update our cache. */
3618	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3619	AssertRCReturn(rc, rc);
3620	pVmcsInfo->u32PinCtls = fVal;
3621
3622	return VINF_SUCCESS;
3623	}
3624
3625
3626	/**
3627	* Sets up secondary processor-based VM-execution controls in the VMCS.
3628	*
3629	* @returns VBox status code.
3630	* @param pVCpu The cross context virtual CPU structure.
3631	* @param pVmcsInfo The VMCS info. object.
3632	*/
3633	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3634	{
3635	PVM pVM = pVCpu->CTX_SUFF(pVM);
3636	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3637	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3638
3639	/* WBINVD causes a VM-exit. */
3640	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3641	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3642
3643	/* Enable EPT (aka nested-paging). */
3644	if (pVM->hm.s.fNestedPaging)
3645	fVal \|= VMX_PROC_CTLS2_EPT;
3646
3647	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3648	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3649	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3650	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3651	fVal \|= VMX_PROC_CTLS2_INVPCID;
3652
3653	/* Enable VPID. */
3654	if (pVM->hm.s.vmx.fVpid)
3655	fVal \|= VMX_PROC_CTLS2_VPID;
3656
3657	/* Enable unrestricted guest execution. */
3658	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3659	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3660
3661	#if 0
3662	if (pVM->hm.s.fVirtApicRegs)
3663	{
3664	/* Enable APIC-register virtualization. */
3665	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3666	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3667
3668	/* Enable virtual-interrupt delivery. */
3669	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3670	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3671	}
3672	#endif
3673
3674	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3675	where the TPR shadow resides. */
3676	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3677	* done dynamically. */
3678	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3679	{
3680	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3681	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3682	AssertRCReturn(rc, rc);
3683	}
3684
3685	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3686	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3687	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3688	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3689	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3690
3691	/* Enable Pause-Loop exiting. */
3692	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3693	&& pVM->hm.s.vmx.cPleGapTicks
3694	&& pVM->hm.s.vmx.cPleWindowTicks)
3695	{
3696	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3697
3698	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3699	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3700	AssertRCReturn(rc, rc);
3701	}
3702
3703	if ((fVal & fZap) != fVal)
3704	{
3705	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3706	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3707	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3708	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3709	}
3710
3711	/* Commit it to the VMCS and update our cache. */
3712	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3713	AssertRCReturn(rc, rc);
3714	pVmcsInfo->u32ProcCtls2 = fVal;
3715
3716	return VINF_SUCCESS;
3717	}
3718
3719
3720	/**
3721	* Sets up processor-based VM-execution controls in the VMCS.
3722	*
3723	* @returns VBox status code.
3724	* @param pVCpu The cross context virtual CPU structure.
3725	* @param pVmcsInfo The VMCS info. object.
3726	*/
3727	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3728	{
3729	PVM pVM = pVCpu->CTX_SUFF(pVM);
3730
3731	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3732	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3733
3734	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3735	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3736	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3737	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3738	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3739	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3740	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3741
3742	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3743	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3744	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3745	{
3746	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3747	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3748	}
3749
3750	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3751	if (!pVM->hm.s.fNestedPaging)
3752	{
3753	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3754	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3755	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3756	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3757	}
3758
3759	/* Use TPR shadowing if supported by the CPU. */
3760	if ( PDMHasApic(pVM)
3761	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3762	{
3763	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3764	/* CR8 writes cause a VM-exit based on TPR threshold. */
3765	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3766	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3767	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3768	AssertRCReturn(rc, rc);
3769	}
3770	else
3771	{
3772	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3773	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3774	if (pVM->hm.s.fAllow64BitGuests)
3775	{
3776	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3777	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3778	}
3779	}
3780
3781	/* Use MSR-bitmaps if supported by the CPU. */
3782	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3783	{
3784	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3785	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3786	AssertRCReturn(rc, rc);
3787	}
3788
3789	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3790	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3791	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3792
3793	if ((fVal & fZap) != fVal)
3794	{
3795	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3796	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3797	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3798	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3799	}
3800
3801	/* Commit it to the VMCS and update our cache. */
3802	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3803	AssertRCReturn(rc, rc);
3804	pVmcsInfo->u32ProcCtls = fVal;
3805
3806	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3807	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3808	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3809
3810	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3811	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3812	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3813
3814	/* Sanity check, should not really happen. */
3815	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3816	{ /* likely */ }
3817	else
3818	{
3819	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3820	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3821	}
3822
3823	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3824	return VINF_SUCCESS;
3825	}
3826
3827
3828	/**
3829	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3830	* Processor-based VM-execution) control fields in the VMCS.
3831	*
3832	* @returns VBox status code.
3833	* @param pVCpu The cross context virtual CPU structure.
3834	* @param pVmcsInfo The VMCS info. object.
3835	*/
3836	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3837	{
3838	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3839	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3840	{
3841	int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3842	rc \|= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3843	if (RT_SUCCESS(rc))
3844	{ /* likely */ }
3845	else
3846	{
3847	LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3848	return rc;
3849	}
3850	}
3851	#endif
3852
3853	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3854	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3855	if (RT_SUCCESS(rc))
3856	{
3857	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3858	if (RT_SUCCESS(rc))
3859	{
3860	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3861	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3862	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3863	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3864	if (RT_SUCCESS(rc))
3865	{
3866	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3867	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3868	return VINF_SUCCESS;
3869	}
3870	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3871	}
3872	else
3873	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3874	}
3875	else
3876	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3877	return rc;
3878	}
3879
3880
3881	/**
3882	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3883	*
3884	* We shall setup those exception intercepts that don't change during the
3885	* lifetime of the VM here. The rest are done dynamically while loading the
3886	* guest state.
3887	*
3888	* @returns VBox status code.
3889	* @param pVCpu The cross context virtual CPU structure.
3890	* @param pVmcsInfo The VMCS info. object.
3891	*/
3892	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3893	{
3894	/*
3895	* The following exceptions are always intercepted:
3896	*
3897	* #AC - To prevent the guest from hanging the CPU.
3898	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3899	* recursive #DBs can cause a CPU hang.
3900	* #PF - To sync our shadow page tables when nested-paging is not used.
3901	*/
3902	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3903	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3904	\| RT_BIT(X86_XCPT_DB)
3905	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3906
3907	/* Commit it to the VMCS. */
3908	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3909	AssertRCReturn(rc, rc);
3910
3911	/* Update our cache of the exception bitmap. */
3912	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3913	return VINF_SUCCESS;
3914	}
3915
3916
3917	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3918	/**
3919	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3920	*
3921	* @returns VBox status code.
3922	* @param pVCpu The cross context virtual CPU structure.
3923	* @param pVmcsInfo The VMCS info. object.
3924	*/
3925	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3926	{
3927	PVM pVM = pVCpu->CTX_SUFF(pVM);
3928	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3929	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3930	if (RT_SUCCESS(rc))
3931	{
3932	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3933	if (RT_SUCCESS(rc))
3934	{
3935	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3936	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3937	if (RT_SUCCESS(rc))
3938	{
3939	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3940	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3941	if (RT_SUCCESS(rc))
3942	return VINF_SUCCESS;
3943
3944	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3945	}
3946	else
3947	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3948	}
3949	else
3950	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3951	}
3952	else
3953	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3954
3955	return rc;
3956	}
3957	#endif
3958
3959
3960	/**
3961	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3962	* VMX.
3963	*
3964	* @returns VBox status code.
3965	* @param pVCpu The cross context virtual CPU structure.
3966	* @param pVmcsInfo The VMCS info. object.
3967	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3968	*/
3969	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3970	{
3971	Assert(pVmcsInfo->pvVmcs);
3972	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3973
3974	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3975	PVM pVM = pVCpu->CTX_SUFF(pVM);
3976	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3977	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3978
3979	LogFlowFunc(("\n"));
3980
3981	/*
3982	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3983	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3984	*/
3985	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3986	if (RT_SUCCESS(rc))
3987	{
3988	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3989	if (RT_SUCCESS(rc))
3990	{
3991	if (!fIsNstGstVmcs)
3992	{
3993	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3994	if (RT_SUCCESS(rc))
3995	{
3996	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3997	if (RT_SUCCESS(rc))
3998	{
3999	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4000	if (RT_SUCCESS(rc))
4001	{
4002	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4003	if (RT_SUCCESS(rc))
4004	{
4005	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4006	/*
4007	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4008	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4009	* making it fit for use when VMCS shadowing is later enabled.
4010	*/
4011	if (pVmcsInfo->pvShadowVmcs)
4012	{
4013	VMXVMCSREVID VmcsRevId;
4014	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4015	VmcsRevId.n.fIsShadowVmcs = 1;
4016	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4017	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4018	if (RT_SUCCESS(rc))
4019	{ /* likely */ }
4020	else
4021	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4022	}
4023	#endif
4024	}
4025	else
4026	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4027	}
4028	else
4029	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4030	}
4031	else
4032	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4033	}
4034	else
4035	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4036	}
4037	else
4038	{
4039	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4040	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4041	if (RT_SUCCESS(rc))
4042	{ /* likely */ }
4043	else
4044	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4045	#else
4046	AssertFailed();
4047	#endif
4048	}
4049	}
4050	else
4051	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4052	}
4053	else
4054	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4055
4056	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4057	if (RT_SUCCESS(rc))
4058	{
4059	rc = hmR0VmxClearVmcs(pVmcsInfo);
4060	if (RT_SUCCESS(rc))
4061	{ /* likely */ }
4062	else
4063	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4064	}
4065
4066	/*
4067	* Update the last-error record both for failures and success, so we
4068	* can propagate the status code back to ring-3 for diagnostics.
4069	*/
4070	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4071	NOREF(pszVmcs);
4072	return rc;
4073	}
4074
4075
4076	/**
4077	* Does global VT-x initialization (called during module initialization).
4078	*
4079	* @returns VBox status code.
4080	*/
4081	VMMR0DECL(int) VMXR0GlobalInit(void)
4082	{
4083	#ifdef HMVMX_USE_FUNCTION_TABLE
4084	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4085	# ifdef VBOX_STRICT
4086	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4087	Assert(g_apfnVMExitHandlers[i]);
4088	# endif
4089	#endif
4090	return VINF_SUCCESS;
4091	}
4092
4093
4094	/**
4095	* Does global VT-x termination (called during module termination).
4096	*/
4097	VMMR0DECL(void) VMXR0GlobalTerm()
4098	{
4099	/* Nothing to do currently. */
4100	}
4101
4102
4103	/**
4104	* Sets up and activates VT-x on the current CPU.
4105	*
4106	* @returns VBox status code.
4107	* @param pHostCpu The HM physical-CPU structure.
4108	* @param pVM The cross context VM structure. Can be
4109	* NULL after a host resume operation.
4110	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4111	* fEnabledByHost is @c true).
4112	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4113	* @a fEnabledByHost is @c true).
4114	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4115	* enable VT-x on the host.
4116	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4117	*/
4118	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4119	PCSUPHWVIRTMSRS pHwvirtMsrs)
4120	{
4121	AssertPtr(pHostCpu);
4122	AssertPtr(pHwvirtMsrs);
4123	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4124
4125	/* Enable VT-x if it's not already enabled by the host. */
4126	if (!fEnabledByHost)
4127	{
4128	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4129	if (RT_FAILURE(rc))
4130	return rc;
4131	}
4132
4133	/*
4134	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4135	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4136	* invalidated when flushing by VPID.
4137	*/
4138	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4139	{
4140	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4141	pHostCpu->fFlushAsidBeforeUse = false;
4142	}
4143	else
4144	pHostCpu->fFlushAsidBeforeUse = true;
4145
4146	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4147	++pHostCpu->cTlbFlushes;
4148
4149	return VINF_SUCCESS;
4150	}
4151
4152
4153	/**
4154	* Deactivates VT-x on the current CPU.
4155	*
4156	* @returns VBox status code.
4157	* @param pvCpuPage Pointer to the VMXON region.
4158	* @param HCPhysCpuPage Physical address of the VMXON region.
4159	*
4160	* @remarks This function should never be called when SUPR0EnableVTx() or
4161	* similar was used to enable VT-x on the host.
4162	*/
4163	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4164	{
4165	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4166
4167	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4168	return hmR0VmxLeaveRootMode();
4169	}
4170
4171
4172	/**
4173	* Does per-VM VT-x initialization.
4174	*
4175	* @returns VBox status code.
4176	* @param pVM The cross context VM structure.
4177	*/
4178	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4179	{
4180	AssertPtr(pVM);
4181	LogFlowFunc(("pVM=%p\n", pVM));
4182
4183	int rc = hmR0VmxStructsAlloc(pVM);
4184	if (RT_FAILURE(rc))
4185	{
4186	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4187	return rc;
4188	}
4189
4190	return VINF_SUCCESS;
4191	}
4192
4193
4194	/**
4195	* Does per-VM VT-x termination.
4196	*
4197	* @returns VBox status code.
4198	* @param pVM The cross context VM structure.
4199	*/
4200	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4201	{
4202	AssertPtr(pVM);
4203	LogFlowFunc(("pVM=%p\n", pVM));
4204
4205	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4206	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4207	{
4208	Assert(pVM->hm.s.vmx.pvScratch);
4209	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4210	}
4211	#endif
4212	hmR0VmxStructsFree(pVM);
4213	return VINF_SUCCESS;
4214	}
4215
4216
4217	/**
4218	* Sets up the VM for execution using hardware-assisted VMX.
4219	* This function is only called once per-VM during initialization.
4220	*
4221	* @returns VBox status code.
4222	* @param pVM The cross context VM structure.
4223	*/
4224	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4225	{
4226	AssertPtr(pVM);
4227	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4228
4229	LogFlowFunc(("pVM=%p\n", pVM));
4230
4231	/*
4232	* At least verify if VMX is enabled, since we can't check if we're in
4233	* VMX root mode or not without causing a #GP.
4234	*/
4235	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4236	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4237	{ /* likely */ }
4238	else
4239	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4240
4241	/*
4242	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4243	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4244	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4245	*/
4246	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4247	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4248	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4249	{
4250	LogRelFunc(("Invalid real-on-v86 state.\n"));
4251	return VERR_INTERNAL_ERROR;
4252	}
4253
4254	/* Initialize these always, see hmR3InitFinalizeR0().*/
4255	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4256	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4257
4258	/* Setup the tagged-TLB flush handlers. */
4259	int rc = hmR0VmxSetupTaggedTlb(pVM);
4260	if (RT_FAILURE(rc))
4261	{
4262	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4263	return rc;
4264	}
4265
4266	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4267	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4268	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4269	{
4270	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4271	if (RT_SUCCESS(rc))
4272	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4273	else
4274	{
4275	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4276	return rc;
4277	}
4278	}
4279	#endif
4280
4281	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4282	{
4283	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4284	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4285
4286	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4287	if (RT_SUCCESS(rc))
4288	{
4289	#if HC_ARCH_BITS == 32
4290	hmR0VmxInitVmcsReadCache(pVCpu);
4291	#endif
4292	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4293	if (pVM->cpum.ro.GuestFeatures.fVmx)
4294	{
4295	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4296	if (RT_SUCCESS(rc))
4297	{ /* likely */ }
4298	else
4299	{
4300	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4301	return rc;
4302	}
4303	}
4304	#endif
4305	}
4306	else
4307	{
4308	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4309	return rc;
4310	}
4311	}
4312
4313	return VINF_SUCCESS;
4314	}
4315
4316
4317	#if HC_ARCH_BITS == 32
4318	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4319	/**
4320	* Check if guest state allows safe use of 32-bit switcher again.
4321	*
4322	* Segment bases and protected mode structures must be 32-bit addressable
4323	* because the 32-bit switcher will ignore high dword when writing these VMCS
4324	* fields. See @bugref{8432} for details.
4325	*
4326	* @returns true if safe, false if must continue to use the 64-bit switcher.
4327	* @param pCtx Pointer to the guest-CPU context.
4328	*
4329	* @remarks No-long-jump zone!!!
4330	*/
4331	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4332	{
4333	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4334	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4335	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4336	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4337	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4338	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4339	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4340	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4341	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4342	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4343
4344	/* All good, bases are 32-bit. */
4345	return true;
4346	}
4347	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
4348
4349	# ifdef VBOX_STRICT
4350	static bool hmR0VmxIsValidWriteFieldInCache(uint32_t idxField)
4351	{
4352	switch (idxField)
4353	{
4354	case VMX_VMCS_GUEST_RIP:
4355	case VMX_VMCS_GUEST_RSP:
4356	case VMX_VMCS_GUEST_SYSENTER_EIP:
4357	case VMX_VMCS_GUEST_SYSENTER_ESP:
4358	case VMX_VMCS_GUEST_GDTR_BASE:
4359	case VMX_VMCS_GUEST_IDTR_BASE:
4360	case VMX_VMCS_GUEST_CS_BASE:
4361	case VMX_VMCS_GUEST_DS_BASE:
4362	case VMX_VMCS_GUEST_ES_BASE:
4363	case VMX_VMCS_GUEST_FS_BASE:
4364	case VMX_VMCS_GUEST_GS_BASE:
4365	case VMX_VMCS_GUEST_SS_BASE:
4366	case VMX_VMCS_GUEST_LDTR_BASE:
4367	case VMX_VMCS_GUEST_TR_BASE:
4368	case VMX_VMCS_GUEST_CR3:
4369	return true;
4370	}
4371	return false;
4372	}
4373
4374	static bool hmR0VmxIsValidReadFieldInCache(uint32_t idxField)
4375	{
4376	switch (idxField)
4377	{
4378	/* Read-only fields. */
4379	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4380	return true;
4381	}
4382	/* Remaining readable fields should also be writable. */
4383	return hmR0VmxIsValidWriteFieldInCache(idxField);
4384	}
4385	# endif /* VBOX_STRICT */
4386
4387
4388	/**
4389	* Executes the specified handler in 64-bit mode.
4390	*
4391	* @returns VBox status code (no informational status codes).
4392	* @param pVCpu The cross context virtual CPU structure.
4393	* @param enmOp The operation to perform.
4394	* @param cParams Number of parameters.
4395	* @param paParam Array of 32-bit parameters.
4396	*/
4397	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
4398	{
4399	AssertPtr(pVCpu);
4400	PVM pVM = pVCpu->CTX_SUFF(pVM);
4401	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4402	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4403	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
4404	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
4405
4406	#ifdef VBOX_STRICT
4407	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
4408	Assert(hmR0VmxIsValidWriteFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
4409
4410	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
4411	Assert(hmR0VmxIsValidReadFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
4412	#endif
4413
4414	/* Disable interrupts. */
4415	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
4416
4417	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4418	RTCPUID idHostCpu = RTMpCpuId();
4419	CPUMR0SetLApic(pVCpu, idHostCpu);
4420	#endif
4421
4422	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4423
4424	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4425	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4426
4427	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4428	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4429	hmR0VmxClearVmcs(pVmcsInfo);
4430
4431	/* Leave VMX root mode and disable VMX. */
4432	VMXDisable();
4433	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4434
4435	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4436	CPUMSetHyperEIP(pVCpu, enmOp);
4437	for (int i = (int)cParams - 1; i >= 0; i--)
4438	CPUMPushHyper(pVCpu, paParam[i]);
4439
4440	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4441
4442	/* Call the switcher. */
4443	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
4444	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4445
4446	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
4447	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
4448
4449	/* Re-enter VMX root mode. */
4450	int rc2 = VMXEnable(HCPhysCpuPage);
4451	if (RT_FAILURE(rc2))
4452	{
4453	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4454	ASMSetFlags(fOldEFlags);
4455	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
4456	return rc2;
4457	}
4458
4459	/* Restore the VMCS as the current VMCS. */
4460	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
4461	AssertRC(rc2);
4462	Assert(!(ASMGetFlags() & X86_EFL_IF));
4463	ASMSetFlags(fOldEFlags);
4464	return rc;
4465	}
4466
4467
4468	/**
4469	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4470	* supporting 64-bit guests.
4471	*
4472	* @returns VBox status code.
4473	* @param fResume Whether to VMLAUNCH or VMRESUME.
4474	* @param pCtx Pointer to the guest-CPU context.
4475	* @param pCache Pointer to the VMCS batch cache.
4476	* @param pVM The cross context VM structure.
4477	* @param pVCpu The cross context virtual CPU structure.
4478	*/
4479	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4480	{
4481	NOREF(fResume);
4482
4483	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4484	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4485	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4486
4487	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4488	pCache->uPos = 1;
4489	pCache->interPD = PGMGetInterPaeCR3(pVM);
4490	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4491	#endif
4492
4493	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4494	pCache->TestIn.HCPhysCpuPage = 0;
4495	pCache->TestIn.HCPhysVmcs = 0;
4496	pCache->TestIn.pCache = 0;
4497	pCache->TestOut.HCPhysVmcs = 0;
4498	pCache->TestOut.pCache = 0;
4499	pCache->TestOut.pCtx = 0;
4500	pCache->TestOut.eflags = 0;
4501	#else
4502	NOREF(pCache);
4503	#endif
4504
4505	uint32_t aParam[10];
4506	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4507	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4508	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4509	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4510	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4511	aParam[5] = 0;
4512	aParam[6] = VM_RC_ADDR(pVM, pVM);
4513	aParam[7] = 0;
4514	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4515	aParam[9] = 0;
4516
4517	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4518	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4519	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4520	#endif
4521	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4522
4523	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4524	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4525	Assert(pCtx->dr[4] == 10);
4526	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4527	#endif
4528
4529	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4530	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4531	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4532	pVmcsInfo->HCPhysVmcs));
4533	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4534	pCache->TestOut.HCPhysVmcs));
4535	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4536	pCache->TestOut.pCache));
4537	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4538	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4539	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4540	pCache->TestOut.pCtx));
4541	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4542	#endif
4543	NOREF(pCtx);
4544	return rc;
4545	}
4546	#endif
4547
4548
4549	/**
4550	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4551	* the VMCS.
4552	*
4553	* @returns VBox status code.
4554	*/
4555	static int hmR0VmxExportHostControlRegs(void)
4556	{
4557	RTCCUINTREG uReg = ASMGetCR0();
4558	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4559	AssertRCReturn(rc, rc);
4560
4561	uReg = ASMGetCR3();
4562	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4563	AssertRCReturn(rc, rc);
4564
4565	uReg = ASMGetCR4();
4566	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4567	AssertRCReturn(rc, rc);
4568	return rc;
4569	}
4570
4571
4572	/**
4573	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4574	* the host-state area in the VMCS.
4575	*
4576	* @returns VBox status code.
4577	* @param pVCpu The cross context virtual CPU structure.
4578	*/
4579	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4580	{
4581	#if HC_ARCH_BITS == 64
4582	/**
4583	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4584	* requirements. See hmR0VmxExportHostSegmentRegs().
4585	*/
4586	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4587	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4588	{ \
4589	bool fValidSelector = true; \
4590	if ((selValue) & X86_SEL_LDT) \
4591	{ \
4592	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4593	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4594	} \
4595	if (fValidSelector) \
4596	{ \
4597	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4598	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4599	} \
4600	(selValue) = 0; \
4601	}
4602
4603	/*
4604	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4605	* will be messed up. We should -not- save the messed up state without restoring
4606	* the original host-state, see @bugref{7240}.
4607	*
4608	* This apparently can happen (most likely the FPU changes), deal with it rather than
4609	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4610	*/
4611	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4612	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4613	{
4614	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4615	pVCpu->idCpu));
4616	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4617	}
4618	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4619	#else
4620	RT_NOREF(pVCpu);
4621	#endif
4622
4623	/*
4624	* Host DS, ES, FS and GS segment registers.
4625	*/
4626	#if HC_ARCH_BITS == 64
4627	RTSEL uSelDS = ASMGetDS();
4628	RTSEL uSelES = ASMGetES();
4629	RTSEL uSelFS = ASMGetFS();
4630	RTSEL uSelGS = ASMGetGS();
4631	#else
4632	RTSEL uSelDS = 0;
4633	RTSEL uSelES = 0;
4634	RTSEL uSelFS = 0;
4635	RTSEL uSelGS = 0;
4636	#endif
4637
4638	/*
4639	* Host CS and SS segment registers.
4640	*/
4641	RTSEL uSelCS = ASMGetCS();
4642	RTSEL uSelSS = ASMGetSS();
4643
4644	/*
4645	* Host TR segment register.
4646	*/
4647	RTSEL uSelTR = ASMGetTR();
4648
4649	#if HC_ARCH_BITS == 64
4650	/*
4651	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4652	* gain VM-entry and restore them before we get preempted.
4653	*
4654	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4655	*/
4656	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4657	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4658	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4659	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4660	# undef VMXLOCAL_ADJUST_HOST_SEG
4661	#endif
4662
4663	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4664	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4665	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4666	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4667	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4668	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4669	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4670	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4671	Assert(uSelCS);
4672	Assert(uSelTR);
4673
4674	/* Write these host selector fields into the host-state area in the VMCS. */
4675	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4676	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4677	#if HC_ARCH_BITS == 64
4678	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4679	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4680	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4681	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4682	#else
4683	NOREF(uSelDS);
4684	NOREF(uSelES);
4685	NOREF(uSelFS);
4686	NOREF(uSelGS);
4687	#endif
4688	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4689	AssertRCReturn(rc, rc);
4690
4691	/*
4692	* Host GDTR and IDTR.
4693	*/
4694	RTGDTR Gdtr;
4695	RTIDTR Idtr;
4696	RT_ZERO(Gdtr);
4697	RT_ZERO(Idtr);
4698	ASMGetGDTR(&Gdtr);
4699	ASMGetIDTR(&Idtr);
4700	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4701	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4702	AssertRCReturn(rc, rc);
4703
4704	#if HC_ARCH_BITS == 64
4705	/*
4706	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4707	* them to the maximum limit (0xffff) on every VM-exit.
4708	*/
4709	if (Gdtr.cbGdt != 0xffff)
4710	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4711
4712	/*
4713	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4714	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4715	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4716	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4717	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4718	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4719	* at 0xffff on hosts where we are sure it won't cause trouble.
4720	*/
4721	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4722	if (Idtr.cbIdt < 0x0fff)
4723	# else
4724	if (Idtr.cbIdt != 0xffff)
4725	# endif
4726	{
4727	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4728	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4729	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4730	}
4731	#endif
4732
4733	/*
4734	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4735	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4736	* RPL should be too in most cases.
4737	*/
4738	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4739	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4740
4741	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4742	#if HC_ARCH_BITS == 64
4743	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4744
4745	/*
4746	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4747	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4748	* restoration if the host has something else. Task switching is not supported in 64-bit
4749	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4750	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4751	*
4752	* [1] See Intel spec. 3.5 "System Descriptor Types".
4753	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4754	*/
4755	PVM pVM = pVCpu->CTX_SUFF(pVM);
4756	Assert(pDesc->System.u4Type == 11);
4757	if ( pDesc->System.u16LimitLow != 0x67
4758	\|\| pDesc->System.u4LimitHigh)
4759	{
4760	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4761	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4762	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4763	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4764	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4765	}
4766
4767	/*
4768	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4769	*/
4770	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4771	{
4772	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4773	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4774	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4775	{
4776	/* The GDT is read-only but the writable GDT is available. */
4777	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4778	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4779	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4780	AssertRCReturn(rc, rc);
4781	}
4782	}
4783	#else
4784	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4785	#endif
4786	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4787	AssertRCReturn(rc, rc);
4788
4789	/*
4790	* Host FS base and GS base.
4791	*/
4792	#if HC_ARCH_BITS == 64
4793	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4794	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4795	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4796	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4797	AssertRCReturn(rc, rc);
4798
4799	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4800	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4801	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4802	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4803	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4804	#endif
4805	return VINF_SUCCESS;
4806	}
4807
4808
4809	/**
4810	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4811	* host-state area of the VMCS.
4812	*
4813	* These MSRs will be automatically restored on the host after every successful
4814	* VM-exit.
4815	*
4816	* @returns VBox status code.
4817	* @param pVCpu The cross context virtual CPU structure.
4818	*
4819	* @remarks No-long-jump zone!!!
4820	*/
4821	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4822	{
4823	AssertPtr(pVCpu);
4824
4825	/*
4826	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4827	* rather than swapping them on every VM-entry.
4828	*/
4829	hmR0VmxLazySaveHostMsrs(pVCpu);
4830
4831	/*
4832	* Host Sysenter MSRs.
4833	*/
4834	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4835	#if HC_ARCH_BITS == 32
4836	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4837	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4838	#else
4839	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4840	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4841	#endif
4842	AssertRCReturn(rc, rc);
4843
4844	/*
4845	* Host EFER MSR.
4846	*
4847	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4848	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4849	*/
4850	PVM pVM = pVCpu->CTX_SUFF(pVM);
4851	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4852	{
4853	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4854	AssertRCReturn(rc, rc);
4855	}
4856
4857	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4858	* hmR0VmxExportGuestEntryExitCtls(). */
4859
4860	return VINF_SUCCESS;
4861	}
4862
4863
4864	/**
4865	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4866	*
4867	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4868	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4869	*
4870	* @returns true if we need to load guest EFER, false otherwise.
4871	* @param pVCpu The cross context virtual CPU structure.
4872	*
4873	* @remarks Requires EFER, CR4.
4874	* @remarks No-long-jump zone!!!
4875	*/
4876	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4877	{
4878	#ifdef HMVMX_ALWAYS_SWAP_EFER
4879	RT_NOREF(pVCpu);
4880	return true;
4881	#else
4882	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4883	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4884	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4885	if (CPUMIsGuestInLongModeEx(pCtx))
4886	return false;
4887	#endif
4888
4889	PVM pVM = pVCpu->CTX_SUFF(pVM);
4890	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4891	uint64_t const u64GuestEfer = pCtx->msrEFER;
4892
4893	/*
4894	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4895	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4896	*/
4897	if ( CPUMIsGuestInLongModeEx(pCtx)
4898	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4899	return true;
4900
4901	/*
4902	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4903	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4904	*
4905	* See Intel spec. 4.5 "IA-32e Paging".
4906	* See Intel spec. 4.1.1 "Three Paging Modes".
4907	*
4908	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4909	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4910	*/
4911	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4912	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4913	if ( (pCtx->cr4 & X86_CR4_PAE)
4914	&& (pCtx->cr0 & X86_CR0_PG)
4915	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4916	{
4917	/* Assert that host is NX capable. */
4918	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4919	return true;
4920	}
4921
4922	return false;
4923	#endif
4924	}
4925
4926	/**
4927	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4928	* VMCS.
4929	*
4930	* This is typically required when the guest changes paging mode.
4931	*
4932	* @returns VBox status code.
4933	* @param pVCpu The cross context virtual CPU structure.
4934	* @param pVmxTransient The VMX-transient structure.
4935	*
4936	* @remarks Requires EFER.
4937	* @remarks No-long-jump zone!!!
4938	*/
4939	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4940	{
4941	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4942	{
4943	PVM pVM = pVCpu->CTX_SUFF(pVM);
4944	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4945
4946	/*
4947	* VM-entry controls.
4948	*/
4949	{
4950	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4951	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4952
4953	/*
4954	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4955	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4956	*
4957	* For nested-guests, this is a mandatory VM-entry control. It's also
4958	* required because we do not want to leak host bits to the nested-guest.
4959	*/
4960	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4961
4962	/*
4963	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4964	*
4965	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4966	* required to get the nested-guest working with hardware-assisted VMX execution.
4967	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4968	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4969	* here rather than while merging the guest VMCS controls.
4970	*/
4971	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4972	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4973	else
4974	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4975
4976	/*
4977	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4978	*
4979	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4980	* regardless of whether the nested-guest VMCS specifies it because we are free to
4981	* load whatever MSRs we require and we do not need to modify the guest visible copy
4982	* of the VM-entry MSR load area.
4983	*/
4984	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4985	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4986	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4987	else
4988	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4989
4990	/*
4991	* The following should -not- be set (since we're not in SMM mode):
4992	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4993	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4994	*/
4995
4996	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4997	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4998
4999	if ((fVal & fZap) == fVal)
5000	{ /* likely */ }
5001	else
5002	{
5003	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
5004	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
5005	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
5006	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5007	}
5008
5009	/* Commit it to the VMCS. */
5010	if (pVmcsInfo->u32EntryCtls != fVal)
5011	{
5012	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
5013	AssertRCReturn(rc, rc);
5014	pVmcsInfo->u32EntryCtls = fVal;
5015	}
5016	}
5017
5018	/*
5019	* VM-exit controls.
5020	*/
5021	{
5022	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
5023	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
5024
5025	/*
5026	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
5027	* supported the 1-setting of this bit.
5028	*
5029	* For nested-guests, we set the "save debug controls" as the converse
5030	* "load debug controls" is mandatory for nested-guests anyway.
5031	*/
5032	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
5033
5034	/*
5035	* Set the host long mode active (EFER.LMA) bit (which Intel calls
5036	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
5037	* host EFER.LMA and EFER.LME bit to this value. See assertion in
5038	* hmR0VmxExportHostMsrs().
5039	*
5040	* For nested-guests, we always set this bit as we do not support 32-bit
5041	* hosts.
5042	*/
5043	#if HC_ARCH_BITS == 64
5044	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5045	#else
5046	Assert(!pVmxTransient->fIsNestedGuest);
5047	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
5048	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
5049	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
5050	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
5051	{
5052	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
5053	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5054	}
5055	else
5056	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
5057	#endif
5058
5059	/*
5060	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
5061	*
5062	* For nested-guests, we should use the "save IA32_EFER" control if we also
5063	* used the "load IA32_EFER" control while exporting VM-entry controls.
5064	*/
5065	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
5066	&& hmR0VmxShouldSwapEferMsr(pVCpu))
5067	{
5068	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
5069	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
5070	}
5071
5072	/*
5073	* Enable saving of the VMX-preemption timer value on VM-exit.
5074	* For nested-guests, currently not exposed/used.
5075	*/
5076	if ( pVM->hm.s.vmx.fUsePreemptTimer
5077	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
5078	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
5079
5080	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
5081	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
5082
5083	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
5084	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
5085	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
5086
5087	if ((fVal & fZap) == fVal)
5088	{ /* likely */ }
5089	else
5090	{
5091	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
5092	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
5093	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
5094	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5095	}
5096
5097	/* Commit it to the VMCS. */
5098	if (pVmcsInfo->u32ExitCtls != fVal)
5099	{
5100	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5101	AssertRCReturn(rc, rc);
5102	pVmcsInfo->u32ExitCtls = fVal;
5103	}
5104	}
5105
5106	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5107	}
5108	return VINF_SUCCESS;
5109	}
5110
5111
5112	/**
5113	* Sets the TPR threshold in the VMCS.
5114	*
5115	* @returns VBox status code.
5116	* @param pVCpu The cross context virtual CPU structure.
5117	* @param pVmcsInfo The VMCS info. object.
5118	* @param u32TprThreshold The TPR threshold (task-priority class only).
5119	*/
5120	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5121	{
5122	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5123	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5124	RT_NOREF2(pVCpu, pVmcsInfo);
5125	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5126	}
5127
5128
5129	/**
5130	* Exports the guest APIC TPR state into the VMCS.
5131	*
5132	* @returns VBox status code.
5133	* @param pVCpu The cross context virtual CPU structure.
5134	* @param pVmxTransient The VMX-transient structure.
5135	*
5136	* @remarks No-long-jump zone!!!
5137	*/
5138	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5139	{
5140	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5141	{
5142	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5143
5144	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5145	if (!pVmxTransient->fIsNestedGuest)
5146	{
5147	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5148	&& APICIsEnabled(pVCpu))
5149	{
5150	/*
5151	* Setup TPR shadowing.
5152	*/
5153	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5154	{
5155	bool fPendingIntr = false;
5156	uint8_t u8Tpr = 0;
5157	uint8_t u8PendingIntr = 0;
5158	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5159	AssertRCReturn(rc, rc);
5160
5161	/*
5162	* If there are interrupts pending but masked by the TPR, instruct VT-x to
5163	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5164	* priority of the pending interrupt so we can deliver the interrupt. If there
5165	* are no interrupts pending, set threshold to 0 to not cause any
5166	* TPR-below-threshold VM-exits.
5167	*/
5168	Assert(pVmcsInfo->pbVirtApic);
5169	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
5170	uint32_t u32TprThreshold = 0;
5171	if (fPendingIntr)
5172	{
5173	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5174	(which is the Task-Priority Class). */
5175	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5176	const uint8_t u8TprPriority = u8Tpr >> 4;
5177	if (u8PendingPriority <= u8TprPriority)
5178	u32TprThreshold = u8PendingPriority;
5179	}
5180
5181	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
5182	AssertRCReturn(rc, rc);
5183	}
5184	}
5185	}
5186	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5187	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5188	}
5189	return VINF_SUCCESS;
5190	}
5191
5192
5193	/**
5194	* Gets the guest interruptibility-state.
5195	*
5196	* @returns Guest's interruptibility-state.
5197	* @param pVCpu The cross context virtual CPU structure.
5198	* @param pVmxTransient The VMX-transient structure.
5199	*
5200	* @remarks No-long-jump zone!!!
5201	*/
5202	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5203	{
5204	/*
5205	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5206	*/
5207	uint32_t fIntrState = 0;
5208	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5209	{
5210	/* If inhibition is active, RIP and RFLAGS should've been updated
5211	(i.e. read previously from the VMCS or from ring-3). */
5212	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5213	#ifdef VBOX_STRICT
5214	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
5215	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5216	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
5217	#endif
5218	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5219	{
5220	if (pCtx->eflags.Bits.u1IF)
5221	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5222	else
5223	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5224	}
5225	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5226	{
5227	/*
5228	* We can clear the inhibit force flag as even if we go back to the recompiler
5229	* without executing guest code in VT-x, the flag's condition to be cleared is
5230	* met and thus the cleared state is correct.
5231	*/
5232	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5233	}
5234	}
5235
5236	/*
5237	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5238	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5239	* setting this would block host-NMIs and IRET will not clear the blocking.
5240	*
5241	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5242	*
5243	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5244	*/
5245	if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
5246	&& CPUMIsGuestNmiBlocking(pVCpu))
5247	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5248
5249	return fIntrState;
5250	}
5251
5252
5253	/**
5254	* Exports the exception intercepts required for guest execution in the VMCS.
5255	*
5256	* @returns VBox status code.
5257	* @param pVCpu The cross context virtual CPU structure.
5258	* @param pVmxTransient The VMX-transient structure.
5259	*
5260	* @remarks No-long-jump zone!!!
5261	*/
5262	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5263	{
5264	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5265	{
5266	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5267	if ( !pVmxTransient->fIsNestedGuest
5268	&& pVCpu->hm.s.fGIMTrapXcptUD)
5269	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5270	else
5271	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5272
5273	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5274	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5275	}
5276	return VINF_SUCCESS;
5277	}
5278
5279
5280	/**
5281	* Exports the guest's RIP into the guest-state area in the VMCS.
5282	*
5283	* @returns VBox status code.
5284	* @param pVCpu The cross context virtual CPU structure.
5285	*
5286	* @remarks No-long-jump zone!!!
5287	*/
5288	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5289	{
5290	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5291	{
5292	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5293
5294	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5295	AssertRCReturn(rc, rc);
5296
5297	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5298	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5299	}
5300	return VINF_SUCCESS;
5301	}
5302
5303
5304	/**
5305	* Exports the guest's RSP into the guest-state area in the VMCS.
5306	*
5307	* @returns VBox status code.
5308	* @param pVCpu The cross context virtual CPU structure.
5309	*
5310	* @remarks No-long-jump zone!!!
5311	*/
5312	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5313	{
5314	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5315	{
5316	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5317
5318	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5319	AssertRCReturn(rc, rc);
5320
5321	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5322	}
5323	return VINF_SUCCESS;
5324	}
5325
5326
5327	/**
5328	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5329	*
5330	* @returns VBox status code.
5331	* @param pVCpu The cross context virtual CPU structure.
5332	* @param pVmxTransient The VMX-transient structure.
5333	*
5334	* @remarks No-long-jump zone!!!
5335	*/
5336	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5337	{
5338	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5339	{
5340	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5341
5342	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5343	Let us assert it as such and use 32-bit VMWRITE. */
5344	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5345	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5346	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5347	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5348
5349	/*
5350	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5351	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5352	* can run the real-mode guest code under Virtual 8086 mode.
5353	*/
5354	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5355	if (pVmcsInfo->RealMode.fRealOnV86Active)
5356	{
5357	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5358	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5359	Assert(!pVmxTransient->fIsNestedGuest);
5360	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5361	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5362	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5363	}
5364
5365	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5366	AssertRCReturn(rc, rc);
5367
5368	/*
5369	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5370	*
5371	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5372	* through the hypervisor debugger using EFLAGS.TF.
5373	*/
5374	if ( !pVmxTransient->fIsNestedGuest
5375	&& !pVCpu->hm.s.fSingleInstruction
5376	&& fEFlags.Bits.u1TF)
5377	{
5378	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5379	* premature trips to ring-3 esp since IEM does not yet handle it. */
5380	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5381	AssertRCReturn(rc, rc);
5382	}
5383	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5384	* nested-guest VMCS. */
5385
5386	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5387	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
5388	}
5389	return VINF_SUCCESS;
5390	}
5391
5392
5393	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5394	/**
5395	* Copies the nested-guest VMCS to the shadow VMCS.
5396	*
5397	* @returns VBox status code.
5398	* @param pVCpu The cross context virtual CPU structure.
5399	* @param pVmcsInfo The VMCS info. object.
5400	*
5401	* @remarks No-long-jump zone!!!
5402	*/
5403	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5404	{
5405	PVM pVM = pVCpu->CTX_SUFF(pVM);
5406	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5407
5408	/*
5409	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5410	* current VMCS, as we may try saving guest lazy MSRs.
5411	*
5412	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5413	* calling the import VMCS code which is currently performing the guest MSR reads
5414	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5415	* and the rest of the VMX leave session machinery.
5416	*/
5417	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5418
5419	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5420	if (RT_SUCCESS(rc))
5421	{
5422	/*
5423	* Copy all guest read/write VMCS fields.
5424	*
5425	* We don't check for VMWRITE failures here for performance reasons and
5426	* because they are not expected to fail, barring irrecoverable conditions
5427	* like hardware errors.
5428	*/
5429	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5430	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5431	{
5432	uint64_t u64Val;
5433	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5434	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5435	VMXWriteVmcs64(uVmcsField, u64Val);
5436	}
5437
5438	/*
5439	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5440	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5441	*/
5442	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5443	{
5444	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5445	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5446	{
5447	uint64_t u64Val;
5448	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5449	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5450	VMXWriteVmcs64(uVmcsField, u64Val);
5451	}
5452	}
5453
5454	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5455	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5456	}
5457
5458	ASMSetFlags(fEFlags);
5459	return rc;
5460	}
5461
5462
5463	/**
5464	* Copies the shadow VMCS to the nested-guest VMCS.
5465	*
5466	* @returns VBox status code.
5467	* @param pVCpu The cross context virtual CPU structure.
5468	* @param pVmcsInfo The VMCS info. object.
5469	*
5470	* @remarks Called with interrupts disabled.
5471	*/
5472	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5473	{
5474	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5475	PVM pVM = pVCpu->CTX_SUFF(pVM);
5476	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5477
5478	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5479	if (RT_SUCCESS(rc))
5480	{
5481	/*
5482	* Copy guest read/write fields from the shadow VMCS.
5483	* Guest read-only fields cannot be modified, so no need to copy them.
5484	*
5485	* We don't check for VMREAD failures here for performance reasons and
5486	* because they are not expected to fail, barring irrecoverable conditions
5487	* like hardware errors.
5488	*/
5489	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5490	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5491	{
5492	uint64_t u64Val;
5493	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5494	VMXReadVmcs64(uVmcsField, &u64Val);
5495	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5496	}
5497
5498	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5499	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5500	}
5501	return rc;
5502	}
5503
5504
5505	/**
5506	* Enables VMCS shadowing for the given VMCS info. object.
5507	*
5508	* @param pVCpu The cross context virtual CPU structure.
5509	* @param pVmcsInfo The VMCS info. object.
5510	*
5511	* @remarks No-long-jump zone!!!
5512	*/
5513	static void hmR0VmxEnableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5514	{
5515	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5516
5517	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5518	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5519	{
5520	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5521	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5522	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5523	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5524	AssertRC(rc);
5525
5526	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5527	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5528	Log4Func(("Enabled\n"));
5529	}
5530	}
5531
5532
5533	/**
5534	* Disables VMCS shadowing for the given VMCS info. object.
5535	*
5536	* @param pVCpu The cross context virtual CPU structure.
5537	* @param pVmcsInfo The VMCS info. object.
5538	*
5539	* @remarks No-long-jump zone!!!
5540	*/
5541	static void hmR0VmxDisableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5542	{
5543	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5544
5545	/*
5546	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5547	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5548	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5549	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5550	*
5551	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5552	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5553	*/
5554	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5555	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5556	{
5557	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5558	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5559	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5560	AssertRC(rc);
5561
5562	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5563	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5564	Log4Func(("Disabled\n"));
5565	}
5566	}
5567	#endif
5568
5569
5570	/**
5571	* Exports the guest hardware-virtualization state.
5572	*
5573	* @returns VBox status code.
5574	* @param pVCpu The cross context virtual CPU structure.
5575	* @param pVmxTransient The VMX-transient structure.
5576	*
5577	* @remarks No-long-jump zone!!!
5578	*/
5579	static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5580	{
5581	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5582	{
5583	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5584	/*
5585	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5586	* VMCS shadowing.
5587	*/
5588	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5589	{
5590	/*
5591	* If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5592	* copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5593	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5594	*
5595	* We check for VMX root mode here in case the guest executes VMXOFF without
5596	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5597	* not clear the current VMCS pointer.
5598	*/
5599	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5600	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5601	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5602	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5603	{
5604	/* Paranoia. */
5605	Assert(!pVmxTransient->fIsNestedGuest);
5606
5607	/*
5608	* For performance reasons, also check if the guest hypervisor's current VMCS
5609	* was newly loaded or modified before copying it to the shadow VMCS.
5610	*/
5611	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5612	{
5613	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5614	AssertRCReturn(rc, rc);
5615	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5616	}
5617	hmR0VmxEnableVmcsShadowing(pVCpu, pVmcsInfo);
5618	}
5619	else
5620	hmR0VmxDisableVmcsShadowing(pVCpu, pVmcsInfo);
5621	}
5622	#else
5623	NOREF(pVmxTransient);
5624	#endif
5625	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5626	}
5627	return VINF_SUCCESS;
5628	}
5629
5630
5631	/**
5632	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5633	*
5634	* The guest FPU state is always pre-loaded hence we don't need to bother about
5635	* sharing FPU related CR0 bits between the guest and host.
5636	*
5637	* @returns VBox status code.
5638	* @param pVCpu The cross context virtual CPU structure.
5639	* @param pVmxTransient The VMX-transient structure.
5640	*
5641	* @remarks No-long-jump zone!!!
5642	*/
5643	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5644	{
5645	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5646	{
5647	PVM pVM = pVCpu->CTX_SUFF(pVM);
5648	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5649
5650	/*
5651	* Figure out fixed CR0 bits in VMX operation.
5652	*/
5653	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5654	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5655	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5656	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5657	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5658	else
5659	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5660
5661	if (!pVmxTransient->fIsNestedGuest)
5662	{
5663	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5664	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5665	uint64_t const u64ShadowCr0 = u64GuestCr0;
5666	Assert(!RT_HI_U32(u64GuestCr0));
5667
5668	/*
5669	* Setup VT-x's view of the guest CR0.
5670	*/
5671	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5672	if (pVM->hm.s.fNestedPaging)
5673	{
5674	if (CPUMIsGuestPagingEnabled(pVCpu))
5675	{
5676	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5677	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5678	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5679	}
5680	else
5681	{
5682	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5683	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5684	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5685	}
5686
5687	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5688	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5689	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5690	}
5691	else
5692	{
5693	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5694	u64GuestCr0 \|= X86_CR0_WP;
5695	}
5696
5697	/*
5698	* Guest FPU bits.
5699	*
5700	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5701	* using CR0.TS.
5702	*
5703	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5704	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5705	*/
5706	u64GuestCr0 \|= X86_CR0_NE;
5707
5708	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5709	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5710
5711	/*
5712	* Update exception intercepts.
5713	*/
5714	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5715	if (pVmcsInfo->RealMode.fRealOnV86Active)
5716	{
5717	Assert(PDMVmmDevHeapIsEnabled(pVM));
5718	Assert(pVM->hm.s.vmx.pRealModeTSS);
5719	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5720	}
5721	else
5722	{
5723	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5724	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5725	if (fInterceptMF)
5726	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5727	}
5728
5729	/* Additional intercepts for debugging, define these yourself explicitly. */
5730	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5731	uXcptBitmap \|= 0
5732	\| RT_BIT(X86_XCPT_BP)
5733	\| RT_BIT(X86_XCPT_DE)
5734	\| RT_BIT(X86_XCPT_NM)
5735	\| RT_BIT(X86_XCPT_TS)
5736	\| RT_BIT(X86_XCPT_UD)
5737	\| RT_BIT(X86_XCPT_NP)
5738	\| RT_BIT(X86_XCPT_SS)
5739	\| RT_BIT(X86_XCPT_GP)
5740	\| RT_BIT(X86_XCPT_PF)
5741	\| RT_BIT(X86_XCPT_MF)
5742	;
5743	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5744	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5745	#endif
5746	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5747	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5748	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5749
5750	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5751	u64GuestCr0 \|= fSetCr0;
5752	u64GuestCr0 &= fZapCr0;
5753	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5754
5755	/* Commit the CR0 and related fields to the guest VMCS. */
5756	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5757	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5758	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5759	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5760	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5761	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5762	AssertRCReturn(rc, rc);
5763
5764	/* Update our caches. */
5765	pVmcsInfo->u32ProcCtls = uProcCtls;
5766	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5767
5768	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5769	}
5770	else
5771	{
5772	/*
5773	* With nested-guests, we may have extended the guest/host mask here since we
5774	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5775	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5776	* originally supplied. We must copy those bits from the nested-guest CR0 into
5777	* the nested-guest CR0 read-shadow.
5778	*/
5779	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5780	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5781	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5782	Assert(!RT_HI_U32(u64GuestCr0));
5783	Assert(u64GuestCr0 & X86_CR0_NE);
5784
5785	/*
5786	* Apply the hardware specified fixed CR0 bits and enable caching.
5787	* Note! We could be altering our VMX emulation's fixed bits. We thus
5788	* need to re-apply them while importing CR0.
5789	*/
5790	u64GuestCr0 \|= fSetCr0;
5791	u64GuestCr0 &= fZapCr0;
5792	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5793
5794	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5795	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5796	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5797	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5798	AssertRCReturn(rc, rc);
5799
5800	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5801	}
5802
5803	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5804	}
5805
5806	return VINF_SUCCESS;
5807	}
5808
5809
5810	/**
5811	* Exports the guest control registers (CR3, CR4) into the guest-state area
5812	* in the VMCS.
5813	*
5814	* @returns VBox strict status code.
5815	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5816	* without unrestricted guest access and the VMMDev is not presently
5817	* mapped (e.g. EFI32).
5818	*
5819	* @param pVCpu The cross context virtual CPU structure.
5820	* @param pVmxTransient The VMX-transient structure.
5821	*
5822	* @remarks No-long-jump zone!!!
5823	*/
5824	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5825	{
5826	int rc = VINF_SUCCESS;
5827	PVM pVM = pVCpu->CTX_SUFF(pVM);
5828
5829	/*
5830	* Guest CR2.
5831	* It's always loaded in the assembler code. Nothing to do here.
5832	*/
5833
5834	/*
5835	* Guest CR3.
5836	*/
5837	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5838	{
5839	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5840
5841	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5842	if (pVM->hm.s.fNestedPaging)
5843	{
5844	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5845	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5846
5847	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5848	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5849	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5850	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5851
5852	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5853	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5854	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5855
5856	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5857	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5858	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5859	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5860	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5861	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5862	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5863
5864	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5865	AssertRCReturn(rc, rc);
5866
5867	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5868	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5869	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5870	{
5871	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5872	if (CPUMIsGuestInPAEModeEx(pCtx))
5873	{
5874	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5875	AssertRCReturn(rc, rc);
5876	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5877	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5878	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5879	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5880	AssertRCReturn(rc, rc);
5881	}
5882
5883	/*
5884	* The guest's view of its CR3 is unblemished with nested paging when the
5885	* guest is using paging or we have unrestricted guest execution to handle
5886	* the guest when it's not using paging.
5887	*/
5888	GCPhysGuestCr3 = pCtx->cr3;
5889	}
5890	else
5891	{
5892	/*
5893	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5894	* thinks it accesses physical memory directly, we use our identity-mapped
5895	* page table to map guest-linear to guest-physical addresses. EPT takes care
5896	* of translating it to host-physical addresses.
5897	*/
5898	RTGCPHYS GCPhys;
5899	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5900
5901	/* We obtain it here every time as the guest could have relocated this PCI region. */
5902	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5903	if (RT_SUCCESS(rc))
5904	{ /* likely */ }
5905	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5906	{
5907	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5908	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5909	}
5910	else
5911	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5912
5913	GCPhysGuestCr3 = GCPhys;
5914	}
5915
5916	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5917	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5918	AssertRCReturn(rc, rc);
5919	}
5920	else
5921	{
5922	/* Non-nested paging case, just use the hypervisor's CR3. */
5923	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5924
5925	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5926	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5927	AssertRCReturn(rc, rc);
5928	}
5929
5930	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5931	}
5932
5933	/*
5934	* Guest CR4.
5935	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5936	*/
5937	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5938	{
5939	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5940	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5941
5942	/*
5943	* Figure out fixed CR4 bits in VMX operation.
5944	*/
5945	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5946	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5947	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5948
5949	/*
5950	* With nested-guests, we may have extended the guest/host mask here (since we
5951	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5952	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5953	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5954	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5955	*/
5956	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5957	uint64_t u64GuestCr4 = pCtx->cr4;
5958	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5959	? pCtx->cr4
5960	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5961	Assert(!RT_HI_U32(u64GuestCr4));
5962
5963	/*
5964	* Setup VT-x's view of the guest CR4.
5965	*
5966	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5967	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5968	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5969	*
5970	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5971	*/
5972	if (pVmcsInfo->RealMode.fRealOnV86Active)
5973	{
5974	Assert(pVM->hm.s.vmx.pRealModeTSS);
5975	Assert(PDMVmmDevHeapIsEnabled(pVM));
5976	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5977	}
5978
5979	if (pVM->hm.s.fNestedPaging)
5980	{
5981	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5982	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5983	{
5984	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5985	u64GuestCr4 \|= X86_CR4_PSE;
5986	/* Our identity mapping is a 32-bit page directory. */
5987	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5988	}
5989	/* else use guest CR4.*/
5990	}
5991	else
5992	{
5993	Assert(!pVmxTransient->fIsNestedGuest);
5994
5995	/*
5996	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5997	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5998	*/
5999	switch (pVCpu->hm.s.enmShadowMode)
6000	{
6001	case PGMMODE_REAL: /* Real-mode. */
6002	case PGMMODE_PROTECTED: /* Protected mode without paging. */
6003	case PGMMODE_32_BIT: /* 32-bit paging. */
6004	{
6005	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
6006	break;
6007	}
6008
6009	case PGMMODE_PAE: /* PAE paging. */
6010	case PGMMODE_PAE_NX: /* PAE paging with NX. */
6011	{
6012	u64GuestCr4 \|= X86_CR4_PAE;
6013	break;
6014	}
6015
6016	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
6017	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
6018	#ifdef VBOX_ENABLE_64_BITS_GUESTS
6019	break;
6020	#endif
6021	default:
6022	AssertFailed();
6023	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6024	}
6025	}
6026
6027	/*
6028	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
6029	* Note! For nested-guests, we could be altering our VMX emulation's
6030	* fixed bits. We thus need to re-apply them while importing CR4.
6031	*/
6032	u64GuestCr4 \|= fSetCr4;
6033	u64GuestCr4 &= fZapCr4;
6034
6035	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
6036	/** @todo Fix to 64-bit when we drop 32-bit. */
6037	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
6038	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
6039	AssertRCReturn(rc, rc);
6040
6041	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
6042	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6043
6044	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
6045
6046	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
6047	}
6048	return rc;
6049	}
6050
6051
6052	/**
6053	* Exports the guest debug registers into the guest-state area in the VMCS.
6054	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
6055	*
6056	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
6057	*
6058	* @returns VBox status code.
6059	* @param pVCpu The cross context virtual CPU structure.
6060	* @param pVmxTransient The VMX-transient structure.
6061	*
6062	* @remarks No-long-jump zone!!!
6063	*/
6064	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6065	{
6066	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6067
6068	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
6069	* stepping. */
6070	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6071	if (pVmxTransient->fIsNestedGuest)
6072	{
6073	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
6074	AssertRCReturn(rc, rc);
6075	return VINF_SUCCESS;
6076	}
6077
6078	#ifdef VBOX_STRICT
6079	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
6080	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
6081	{
6082	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
6083	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
6084	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
6085	}
6086	#endif
6087
6088	bool fSteppingDB = false;
6089	bool fInterceptMovDRx = false;
6090	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6091	if (pVCpu->hm.s.fSingleInstruction)
6092	{
6093	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
6094	PVM pVM = pVCpu->CTX_SUFF(pVM);
6095	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
6096	{
6097	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
6098	Assert(fSteppingDB == false);
6099	}
6100	else
6101	{
6102	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
6103	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
6104	pVCpu->hm.s.fClearTrapFlag = true;
6105	fSteppingDB = true;
6106	}
6107	}
6108
6109	uint32_t u32GuestDr7;
6110	if ( fSteppingDB
6111	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
6112	{
6113	/*
6114	* Use the combined guest and host DRx values found in the hypervisor register set
6115	* because the hypervisor debugger has breakpoints active or someone is single stepping
6116	* on the host side without a monitor trap flag.
6117	*
6118	* Note! DBGF expects a clean DR6 state before executing guest code.
6119	*/
6120	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6121	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6122	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
6123	{
6124	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6125	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
6126	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
6127	}
6128	else
6129	#endif
6130	if (!CPUMIsHyperDebugStateActive(pVCpu))
6131	{
6132	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6133	Assert(CPUMIsHyperDebugStateActive(pVCpu));
6134	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6135	}
6136
6137	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
6138	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
6139	pVCpu->hm.s.fUsingHyperDR7 = true;
6140	fInterceptMovDRx = true;
6141	}
6142	else
6143	{
6144	/*
6145	* If the guest has enabled debug registers, we need to load them prior to
6146	* executing guest code so they'll trigger at the right time.
6147	*/
6148	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
6149	{
6150	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6151	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6152	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
6153	{
6154	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6155	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
6156	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
6157	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6158	}
6159	else
6160	#endif
6161	if (!CPUMIsGuestDebugStateActive(pVCpu))
6162	{
6163	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6164	Assert(CPUMIsGuestDebugStateActive(pVCpu));
6165	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6166	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6167	}
6168	Assert(!fInterceptMovDRx);
6169	}
6170	/*
6171	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
6172	* must intercept #DB in order to maintain a correct DR6 guest value, and
6173	* because we need to intercept it to prevent nested #DBs from hanging the
6174	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
6175	*/
6176	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6177	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
6178	&& !CPUMIsGuestDebugStateActive(pVCpu))
6179	#else
6180	else if (!CPUMIsGuestDebugStateActive(pVCpu))
6181	#endif
6182	{
6183	fInterceptMovDRx = true;
6184	}
6185
6186	/* Update DR7 with the actual guest value. */
6187	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
6188	pVCpu->hm.s.fUsingHyperDR7 = false;
6189	}
6190
6191	if (fInterceptMovDRx)
6192	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
6193	else
6194	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
6195
6196	/*
6197	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
6198	* monitor-trap flag and update our cache.
6199	*/
6200	if (uProcCtls != pVmcsInfo->u32ProcCtls)
6201	{
6202	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6203	AssertRCReturn(rc2, rc2);
6204	pVmcsInfo->u32ProcCtls = uProcCtls;
6205	}
6206
6207	/*
6208	* Update guest DR7.
6209	*/
6210	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
6211	AssertRCReturn(rc, rc);
6212
6213	/*
6214	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6215	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6216	*
6217	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6218	*/
6219	if (fSteppingDB)
6220	{
6221	Assert(pVCpu->hm.s.fSingleInstruction);
6222	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6223
6224	uint32_t fIntrState = 0;
6225	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6226	AssertRCReturn(rc, rc);
6227
6228	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6229	{
6230	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6231	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6232	AssertRCReturn(rc, rc);
6233	}
6234	}
6235
6236	return VINF_SUCCESS;
6237	}
6238
6239
6240	#ifdef VBOX_STRICT
6241	/**
6242	* Strict function to validate segment registers.
6243	*
6244	* @param pVCpu The cross context virtual CPU structure.
6245	* @param pVmcsInfo The VMCS info. object.
6246	*
6247	* @remarks Will import guest CR0 on strict builds during validation of
6248	* segments.
6249	*/
6250	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
6251	{
6252	/*
6253	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6254	*
6255	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6256	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6257	* unusable bit and doesn't change the guest-context value.
6258	*/
6259	PVM pVM = pVCpu->CTX_SUFF(pVM);
6260	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6261	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6262	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6263	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6264	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6265	{
6266	/* Protected mode checks */
6267	/* CS */
6268	Assert(pCtx->cs.Attr.n.u1Present);
6269	Assert(!(pCtx->cs.Attr.u & 0xf00));
6270	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6271	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6272	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6273	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6274	\|\| (pCtx->cs.Attr.n.u1Granularity));
6275	/* CS cannot be loaded with NULL in protected mode. */
6276	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6277	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6278	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6279	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6280	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6281	else
6282	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6283	/* SS */
6284	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6285	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6286	if ( !(pCtx->cr0 & X86_CR0_PE)
6287	\|\| pCtx->cs.Attr.n.u4Type == 3)
6288	{
6289	Assert(!pCtx->ss.Attr.n.u2Dpl);
6290	}
6291	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6292	{
6293	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6294	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6295	Assert(pCtx->ss.Attr.n.u1Present);
6296	Assert(!(pCtx->ss.Attr.u & 0xf00));
6297	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6298	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6299	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6300	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6301	\|\| (pCtx->ss.Attr.n.u1Granularity));
6302	}
6303	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6304	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6305	{
6306	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6307	Assert(pCtx->ds.Attr.n.u1Present);
6308	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6309	Assert(!(pCtx->ds.Attr.u & 0xf00));
6310	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6311	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6312	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6313	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6314	\|\| (pCtx->ds.Attr.n.u1Granularity));
6315	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6316	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6317	}
6318	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6319	{
6320	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6321	Assert(pCtx->es.Attr.n.u1Present);
6322	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6323	Assert(!(pCtx->es.Attr.u & 0xf00));
6324	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6325	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6326	\|\| !(pCtx->es.Attr.n.u1Granularity));
6327	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6328	\|\| (pCtx->es.Attr.n.u1Granularity));
6329	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6330	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6331	}
6332	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6333	{
6334	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6335	Assert(pCtx->fs.Attr.n.u1Present);
6336	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6337	Assert(!(pCtx->fs.Attr.u & 0xf00));
6338	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6339	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6340	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6341	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6342	\|\| (pCtx->fs.Attr.n.u1Granularity));
6343	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6344	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6345	}
6346	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6347	{
6348	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6349	Assert(pCtx->gs.Attr.n.u1Present);
6350	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6351	Assert(!(pCtx->gs.Attr.u & 0xf00));
6352	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6353	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6354	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6355	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6356	\|\| (pCtx->gs.Attr.n.u1Granularity));
6357	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6358	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6359	}
6360	/* 64-bit capable CPUs. */
6361	# if HC_ARCH_BITS == 64
6362	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6363	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6364	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6365	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6366	# endif
6367	}
6368	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6369	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6370	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6371	{
6372	/* Real and v86 mode checks. */
6373	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6374	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6375	if (pVmcsInfo->RealMode.fRealOnV86Active)
6376	{
6377	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6378	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6379	}
6380	else
6381	{
6382	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6383	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6384	}
6385
6386	/* CS */
6387	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6388	Assert(pCtx->cs.u32Limit == 0xffff);
6389	Assert(u32CSAttr == 0xf3);
6390	/* SS */
6391	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6392	Assert(pCtx->ss.u32Limit == 0xffff);
6393	Assert(u32SSAttr == 0xf3);
6394	/* DS */
6395	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6396	Assert(pCtx->ds.u32Limit == 0xffff);
6397	Assert(u32DSAttr == 0xf3);
6398	/* ES */
6399	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6400	Assert(pCtx->es.u32Limit == 0xffff);
6401	Assert(u32ESAttr == 0xf3);
6402	/* FS */
6403	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6404	Assert(pCtx->fs.u32Limit == 0xffff);
6405	Assert(u32FSAttr == 0xf3);
6406	/* GS */
6407	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6408	Assert(pCtx->gs.u32Limit == 0xffff);
6409	Assert(u32GSAttr == 0xf3);
6410	/* 64-bit capable CPUs. */
6411	# if HC_ARCH_BITS == 64
6412	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6413	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6414	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6415	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6416	# endif
6417	}
6418	}
6419	#endif /* VBOX_STRICT */
6420
6421
6422	/**
6423	* Exports a guest segment register into the guest-state area in the VMCS.
6424	*
6425	* @returns VBox status code.
6426	* @param pVCpu The cross context virtual CPU structure.
6427	* @param pVmcsInfo The VMCS info. object.
6428	* @param iSegReg The segment register number (X86_SREG_XXX).
6429	* @param pSelReg Pointer to the segment selector.
6430	*
6431	* @remarks No-long-jump zone!!!
6432	*/
6433	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6434	{
6435	Assert(iSegReg < X86_SREG_COUNT);
6436	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6437	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6438	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6439	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6440
6441	uint32_t u32Access = pSelReg->Attr.u;
6442	if (pVmcsInfo->RealMode.fRealOnV86Active)
6443	{
6444	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6445	u32Access = 0xf3;
6446	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6447	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6448	RT_NOREF_PV(pVCpu);
6449	}
6450	else
6451	{
6452	/*
6453	* The way to differentiate between whether this is really a null selector or was just
6454	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6455	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6456	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6457	* NULL selectors loaded in protected-mode have their attribute as 0.
6458	*/
6459	if (!u32Access)
6460	u32Access = X86DESCATTR_UNUSABLE;
6461	}
6462
6463	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6464	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6465	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6466
6467	/*
6468	* Commit it to the VMCS.
6469	*/
6470	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6471	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6472	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6473	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6474	AssertRCReturn(rc, rc);
6475	return rc;
6476	}
6477
6478
6479	/**
6480	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6481	* area in the VMCS.
6482	*
6483	* @returns VBox status code.
6484	* @param pVCpu The cross context virtual CPU structure.
6485	* @param pVmxTransient The VMX-transient structure.
6486	*
6487	* @remarks Will import guest CR0 on strict builds during validation of
6488	* segments.
6489	* @remarks No-long-jump zone!!!
6490	*/
6491	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6492	{
6493	int rc = VERR_INTERNAL_ERROR_5;
6494	PVM pVM = pVCpu->CTX_SUFF(pVM);
6495	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6496	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6497
6498	/*
6499	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6500	*/
6501	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6502	{
6503	#ifdef VBOX_WITH_REM
6504	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6505	{
6506	Assert(!pVmxTransient->fIsNestedGuest);
6507	Assert(pVM->hm.s.vmx.pRealModeTSS);
6508	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6509	if ( pVmcsInfo->fWasInRealMode
6510	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6511	{
6512	/*
6513	* Notify the recompiler must flush its code-cache as the guest -may-
6514	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6515	*/
6516	REMFlushTBs(pVM);
6517	Log4Func(("Switch to protected mode detected!\n"));
6518	pVmcsInfo->fWasInRealMode = false;
6519	}
6520	}
6521	#endif
6522	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6523	{
6524	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6525	if (pVmcsInfo->RealMode.fRealOnV86Active)
6526	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6527	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6528	AssertRCReturn(rc, rc);
6529	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6530	}
6531
6532	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6533	{
6534	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6535	if (pVmcsInfo->RealMode.fRealOnV86Active)
6536	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6537	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6538	AssertRCReturn(rc, rc);
6539	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6540	}
6541
6542	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6543	{
6544	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6545	if (pVmcsInfo->RealMode.fRealOnV86Active)
6546	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6547	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6548	AssertRCReturn(rc, rc);
6549	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6550	}
6551
6552	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6553	{
6554	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6555	if (pVmcsInfo->RealMode.fRealOnV86Active)
6556	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6557	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6558	AssertRCReturn(rc, rc);
6559	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6560	}
6561
6562	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6563	{
6564	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6565	if (pVmcsInfo->RealMode.fRealOnV86Active)
6566	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6567	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6568	AssertRCReturn(rc, rc);
6569	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6570	}
6571
6572	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6573	{
6574	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6575	if (pVmcsInfo->RealMode.fRealOnV86Active)
6576	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6577	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6578	AssertRCReturn(rc, rc);
6579	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6580	}
6581
6582	#ifdef VBOX_STRICT
6583	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6584	#endif
6585	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6586	pCtx->cs.Attr.u));
6587	}
6588
6589	/*
6590	* Guest TR.
6591	*/
6592	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6593	{
6594	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6595
6596	/*
6597	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6598	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6599	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6600	*/
6601	uint16_t u16Sel;
6602	uint32_t u32Limit;
6603	uint64_t u64Base;
6604	uint32_t u32AccessRights;
6605	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6606	{
6607	u16Sel = pCtx->tr.Sel;
6608	u32Limit = pCtx->tr.u32Limit;
6609	u64Base = pCtx->tr.u64Base;
6610	u32AccessRights = pCtx->tr.Attr.u;
6611	}
6612	else
6613	{
6614	Assert(!pVmxTransient->fIsNestedGuest);
6615	Assert(pVM->hm.s.vmx.pRealModeTSS);
6616	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6617
6618	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6619	RTGCPHYS GCPhys;
6620	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6621	AssertRCReturn(rc, rc);
6622
6623	X86DESCATTR DescAttr;
6624	DescAttr.u = 0;
6625	DescAttr.n.u1Present = 1;
6626	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6627
6628	u16Sel = 0;
6629	u32Limit = HM_VTX_TSS_SIZE;
6630	u64Base = GCPhys;
6631	u32AccessRights = DescAttr.u;
6632	}
6633
6634	/* Validate. */
6635	Assert(!(u16Sel & RT_BIT(2)));
6636	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6637	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6638	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6639	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6640	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6641	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6642	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6643	Assert( (u32Limit & 0xfff) == 0xfff
6644	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6645	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6646	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6647
6648	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6649	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6650	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6651	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6652	AssertRCReturn(rc, rc);
6653
6654	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6655	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6656	}
6657
6658	/*
6659	* Guest GDTR.
6660	*/
6661	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6662	{
6663	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6664
6665	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6666	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6667	AssertRCReturn(rc, rc);
6668
6669	/* Validate. */
6670	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6671
6672	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6673	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6674	}
6675
6676	/*
6677	* Guest LDTR.
6678	*/
6679	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6680	{
6681	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6682
6683	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6684	uint32_t u32Access;
6685	if ( !pVmxTransient->fIsNestedGuest
6686	&& !pCtx->ldtr.Attr.u)
6687	u32Access = X86DESCATTR_UNUSABLE;
6688	else
6689	u32Access = pCtx->ldtr.Attr.u;
6690
6691	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6692	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6693	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6694	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6695	AssertRCReturn(rc, rc);
6696
6697	/* Validate. */
6698	if (!(u32Access & X86DESCATTR_UNUSABLE))
6699	{
6700	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6701	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6702	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6703	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6704	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6705	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6706	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6707	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6708	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6709	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6710	}
6711
6712	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6713	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6714	}
6715
6716	/*
6717	* Guest IDTR.
6718	*/
6719	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6720	{
6721	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6722
6723	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6724	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6725	AssertRCReturn(rc, rc);
6726
6727	/* Validate. */
6728	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6729
6730	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6731	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6732	}
6733
6734	return VINF_SUCCESS;
6735	}
6736
6737
6738	/**
6739	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6740	* areas.
6741	*
6742	* These MSRs will automatically be loaded to the host CPU on every successful
6743	* VM-entry and stored from the host CPU on every successful VM-exit.
6744	*
6745	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6746	* actual host MSR values are not- updated here for performance reasons. See
6747	* hmR0VmxExportHostMsrs().
6748	*
6749	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6750	*
6751	* @returns VBox status code.
6752	* @param pVCpu The cross context virtual CPU structure.
6753	* @param pVmxTransient The VMX-transient structure.
6754	*
6755	* @remarks No-long-jump zone!!!
6756	*/
6757	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6758	{
6759	AssertPtr(pVCpu);
6760	AssertPtr(pVmxTransient);
6761
6762	PVM pVM = pVCpu->CTX_SUFF(pVM);
6763	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6764
6765	/*
6766	* MSRs that we use the auto-load/store MSR area in the VMCS.
6767	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6768	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6769	*
6770	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6771	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6772	* emulation, nothing to do here.
6773	*/
6774	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6775	{
6776	if ( !pVmxTransient->fIsNestedGuest
6777	&& pVM->hm.s.fAllow64BitGuests)
6778	{
6779	#if HC_ARCH_BITS == 32
6780	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6781	Assert(!pVmxTransient->fIsNestedGuest);
6782
6783	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6784	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6785	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6786	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6787	AssertRCReturn(rc, rc);
6788	#endif
6789	}
6790	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6791	}
6792
6793	/*
6794	* Guest Sysenter MSRs.
6795	*/
6796	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6797	{
6798	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6799
6800	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6801	{
6802	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6803	AssertRCReturn(rc, rc);
6804	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6805	}
6806
6807	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6808	{
6809	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6810	AssertRCReturn(rc, rc);
6811	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6812	}
6813
6814	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6815	{
6816	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6817	AssertRCReturn(rc, rc);
6818	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6819	}
6820	}
6821
6822	/*
6823	* Guest/host EFER MSR.
6824	*/
6825	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6826	{
6827	/* Whether we are using the VMCS to swap the EFER MSR must have been
6828	determined earlier while exporting VM-entry/VM-exit controls. */
6829	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6830	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6831
6832	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6833	{
6834	/*
6835	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6836	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6837	*/
6838	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6839	{
6840	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6841	AssertRCReturn(rc, rc);
6842	}
6843	else
6844	{
6845	/*
6846	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6847	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6848	*/
6849	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6850	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6851	AssertRCReturn(rc, rc);
6852	}
6853	}
6854	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6855	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6856
6857	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6858	}
6859
6860	/*
6861	* Other MSRs.
6862	* Speculation Control (R/W).
6863	*/
6864	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6865	{
6866	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6867	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6868	{
6869	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6870	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6871	AssertRCReturn(rc, rc);
6872	}
6873	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6874	}
6875
6876	return VINF_SUCCESS;
6877	}
6878
6879
6880	/**
6881	* Selects up the appropriate function to run guest code.
6882	*
6883	* @returns VBox status code.
6884	* @param pVCpu The cross context virtual CPU structure.
6885	* @param pVmxTransient The VMX-transient structure.
6886	*
6887	* @remarks No-long-jump zone!!!
6888	*/
6889	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6890	{
6891	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6892	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6893
6894	if (CPUMIsGuestInLongModeEx(pCtx))
6895	{
6896	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6897	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6898	#endif
6899	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6900	#if HC_ARCH_BITS == 32
6901	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6902	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6903	{
6904	#ifdef VBOX_STRICT
6905	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6906	{
6907	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6908	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6909	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6910	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6911	("fCtxChanged=%#RX64\n", fCtxChanged));
6912	}
6913	#endif
6914	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6915
6916	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6917	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6918	pVmcsInfo->fSwitchedTo64on32 = true;
6919	Log4Func(("Selected 64-bit switcher\n"));
6920	}
6921	#else
6922	/* 64-bit host. */
6923	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6924	#endif
6925	}
6926	else
6927	{
6928	/* Guest is not in long mode, use the 32-bit handler. */
6929	#if HC_ARCH_BITS == 32
6930	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6931	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6932	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6933	{
6934	# ifdef VBOX_STRICT
6935	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6936	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6937	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6938	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6939	("fCtxChanged=%#RX64\n", fCtxChanged));
6940	# endif
6941	}
6942	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6943	/*
6944	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6945	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6946	* switcher flag now because we know the guest is in a sane state where it's safe
6947	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6948	* the much faster 32-bit switcher again.
6949	*/
6950	if (!pVmcsInfo->fSwitchedTo64on32)
6951	{
6952	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6953	Log4Func(("Selected 32-bit switcher\n"));
6954	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6955	}
6956	else
6957	{
6958	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6959	if ( pVmcsInfo->RealMode.fRealOnV86Active
6960	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6961	{
6962	pVmcsInfo->fSwitchedTo64on32 = false;
6963	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6964	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6965	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6966	\| HM_CHANGED_HOST_CONTEXT);
6967	Log4Func(("Selected 32-bit switcher (safe)\n"));
6968	}
6969	}
6970	# else
6971	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6972	# endif
6973	#else
6974	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6975	#endif
6976	}
6977	Assert(pVmcsInfo->pfnStartVM);
6978	return VINF_SUCCESS;
6979	}
6980
6981
6982	/**
6983	* Wrapper for running the guest code in VT-x.
6984	*
6985	* @returns VBox status code, no informational status codes.
6986	* @param pVCpu The cross context virtual CPU structure.
6987	* @param pVmxTransient The VMX-transient structure.
6988	*
6989	* @remarks No-long-jump zone!!!
6990	*/
6991	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6992	{
6993	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6994	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6995	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6996
6997	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6998
6999	/*
7000	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
7001	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
7002	* callee-saved and thus the need for this XMM wrapper.
7003	*
7004	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
7005	*/
7006	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7007	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
7008	PVM pVM = pVCpu->CTX_SUFF(pVM);
7009	#ifdef VBOX_WITH_KERNEL_USING_XMM
7010	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
7011	#else
7012	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
7013	#endif
7014	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
7015	return rc;
7016	}
7017
7018
7019	/**
7020	* Reports world-switch error and dumps some useful debug info.
7021	*
7022	* @param pVCpu The cross context virtual CPU structure.
7023	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
7024	* @param pVmxTransient The VMX-transient structure (only
7025	* exitReason updated).
7026	*/
7027	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
7028	{
7029	Assert(pVCpu);
7030	Assert(pVmxTransient);
7031	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7032
7033	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
7034	switch (rcVMRun)
7035	{
7036	case VERR_VMX_INVALID_VMXON_PTR:
7037	AssertFailed();
7038	break;
7039	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
7040	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
7041	{
7042	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
7043	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
7044	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
7045	AssertRC(rc);
7046
7047	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7048	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
7049	Cannot do it here as we may have been long preempted. */
7050
7051	#ifdef VBOX_STRICT
7052	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7053	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
7054	pVmxTransient->uExitReason));
7055	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
7056	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
7057	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
7058	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
7059	else
7060	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
7061	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
7062	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
7063
7064	/* VMX control bits. */
7065	uint32_t u32Val;
7066	uint64_t u64Val;
7067	RTHCUINTREG uHCReg;
7068	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
7069	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
7070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
7071	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
7072	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
7073	{
7074	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
7075	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
7076	}
7077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
7078	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
7079	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
7080	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
7081	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
7082	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
7083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
7084	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
7085	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
7086	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
7087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
7088	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
7089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
7090	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
7091	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
7092	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
7093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7094	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
7095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7096	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
7097	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
7098	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
7099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
7100	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
7101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
7102	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
7103	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7104	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7105	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7106	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7107	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7108	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7109	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7110	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7111	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7112	{
7113	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7114	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7115	}
7116
7117	/* Guest bits. */
7118	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
7119	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
7120	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
7121	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
7122	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
7123	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
7124	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
7125	{
7126	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
7127	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
7128	}
7129
7130	/* Host bits. */
7131	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
7132	Log4(("Host CR0 %#RHr\n", uHCReg));
7133	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
7134	Log4(("Host CR3 %#RHr\n", uHCReg));
7135	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
7136	Log4(("Host CR4 %#RHr\n", uHCReg));
7137
7138	RTGDTR HostGdtr;
7139	PCX86DESCHC pDesc;
7140	ASMGetGDTR(&HostGdtr);
7141	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
7142	Log4(("Host CS %#08x\n", u32Val));
7143	if (u32Val < HostGdtr.cbGdt)
7144	{
7145	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7146	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
7147	}
7148
7149	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
7150	Log4(("Host DS %#08x\n", u32Val));
7151	if (u32Val < HostGdtr.cbGdt)
7152	{
7153	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7154	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
7155	}
7156
7157	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
7158	Log4(("Host ES %#08x\n", u32Val));
7159	if (u32Val < HostGdtr.cbGdt)
7160	{
7161	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7162	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
7163	}
7164
7165	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
7166	Log4(("Host FS %#08x\n", u32Val));
7167	if (u32Val < HostGdtr.cbGdt)
7168	{
7169	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7170	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
7171	}
7172
7173	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
7174	Log4(("Host GS %#08x\n", u32Val));
7175	if (u32Val < HostGdtr.cbGdt)
7176	{
7177	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7178	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
7179	}
7180
7181	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
7182	Log4(("Host SS %#08x\n", u32Val));
7183	if (u32Val < HostGdtr.cbGdt)
7184	{
7185	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7186	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
7187	}
7188
7189	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
7190	Log4(("Host TR %#08x\n", u32Val));
7191	if (u32Val < HostGdtr.cbGdt)
7192	{
7193	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7194	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
7195	}
7196
7197	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
7198	Log4(("Host TR Base %#RHv\n", uHCReg));
7199	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
7200	Log4(("Host GDTR Base %#RHv\n", uHCReg));
7201	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
7202	Log4(("Host IDTR Base %#RHv\n", uHCReg));
7203	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
7204	Log4(("Host SYSENTER CS %#08x\n", u32Val));
7205	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
7206	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
7207	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
7208	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
7209	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
7210	Log4(("Host RSP %#RHv\n", uHCReg));
7211	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
7212	Log4(("Host RIP %#RHv\n", uHCReg));
7213	# if HC_ARCH_BITS == 64
7214	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
7215	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
7216	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
7217	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
7218	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
7219	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
7220	# endif
7221	#endif /* VBOX_STRICT */
7222	break;
7223	}
7224
7225	default:
7226	/* Impossible */
7227	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
7228	break;
7229	}
7230	}
7231
7232
7233	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
7234	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
7235	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
7236	# endif
7237
7238	/**
7239	* Initialize the VMCS-Read cache.
7240	*
7241	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
7242	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
7243	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
7244	* (those that have a 32-bit FULL & HIGH part).
7245	*
7246	* @param pVCpu The cross context virtual CPU structure.
7247	*/
7248	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
7249	{
7250	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
7251	do { \
7252	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
7253	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
7254	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
7255	++cReadFields; \
7256	} while (0)
7257
7258	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7259	uint32_t cReadFields = 0;
7260
7261	/*
7262	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
7263	* and serve to indicate exceptions to the rules.
7264	*/
7265
7266	/* Guest-natural selector base fields. */
7267	#if 0
7268	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
7269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
7270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
7271	#endif
7272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
7273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
7274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
7275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
7276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
7277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
7278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
7279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
7280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
7281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
7282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
7283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
7284	#if 0
7285	/* Unused natural width guest-state fields. */
7286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
7287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
7288	#endif
7289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
7290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
7291
7292	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
7293	these 64-bit fields (using "FULL" and "HIGH" fields). */
7294	#if 0
7295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
7296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
7297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
7298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
7299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
7300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
7301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
7302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
7303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
7304	#endif
7305
7306	/* Natural width guest-state fields. */
7307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
7308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
7309
7310	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7311	{
7312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
7313	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
7314	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
7315	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
7316	}
7317	else
7318	{
7319	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
7320	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
7321	}
7322
7323	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
7324	}
7325
7326
7327	/**
7328	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
7329	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
7330	* darwin, running 64-bit guests).
7331	*
7332	* @returns VBox status code.
7333	* @param pVCpu The cross context virtual CPU structure.
7334	* @param idxField The VMCS field encoding.
7335	* @param u64Val 16, 32 or 64-bit value.
7336	*/
7337	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7338	{
7339	AssertPtr(pVCpu);
7340	int rc;
7341	switch (idxField)
7342	{
7343	/*
7344	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
7345	*/
7346	/* 64-bit Control fields. */
7347	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
7348	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
7349	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
7350	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
7351	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
7352	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
7353	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
7354	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
7355	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
7356	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
7357	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
7358	case VMX_VMCS64_CTRL_EPTP_FULL:
7359	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
7360	/* 64-bit Guest-state fields. */
7361	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
7362	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
7363	case VMX_VMCS64_GUEST_PAT_FULL:
7364	case VMX_VMCS64_GUEST_EFER_FULL:
7365	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
7366	case VMX_VMCS64_GUEST_PDPTE0_FULL:
7367	case VMX_VMCS64_GUEST_PDPTE1_FULL:
7368	case VMX_VMCS64_GUEST_PDPTE2_FULL:
7369	case VMX_VMCS64_GUEST_PDPTE3_FULL:
7370	/* 64-bit Host-state fields. */
7371	case VMX_VMCS64_HOST_PAT_FULL:
7372	case VMX_VMCS64_HOST_EFER_FULL:
7373	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
7374	{
7375	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7376	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
7377	break;
7378	}
7379
7380	/*
7381	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
7382	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
7383	*/
7384	/* Natural-width Guest-state fields. */
7385	case VMX_VMCS_GUEST_CR3:
7386	case VMX_VMCS_GUEST_ES_BASE:
7387	case VMX_VMCS_GUEST_CS_BASE:
7388	case VMX_VMCS_GUEST_SS_BASE:
7389	case VMX_VMCS_GUEST_DS_BASE:
7390	case VMX_VMCS_GUEST_FS_BASE:
7391	case VMX_VMCS_GUEST_GS_BASE:
7392	case VMX_VMCS_GUEST_LDTR_BASE:
7393	case VMX_VMCS_GUEST_TR_BASE:
7394	case VMX_VMCS_GUEST_GDTR_BASE:
7395	case VMX_VMCS_GUEST_IDTR_BASE:
7396	case VMX_VMCS_GUEST_RSP:
7397	case VMX_VMCS_GUEST_RIP:
7398	case VMX_VMCS_GUEST_SYSENTER_ESP:
7399	case VMX_VMCS_GUEST_SYSENTER_EIP:
7400	{
7401	if (!(RT_HI_U32(u64Val)))
7402	{
7403	/* If this field is 64-bit, VT-x will zero out the top bits. */
7404	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7405	}
7406	else
7407	{
7408	/* Assert that only the 32->64 switcher case should ever come here. */
7409	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
7410	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
7411	}
7412	break;
7413	}
7414
7415	default:
7416	{
7417	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
7418	pVCpu->hm.s.u32HMError = idxField;
7419	rc = VERR_INVALID_PARAMETER;
7420	break;
7421	}
7422	}
7423	AssertRCReturn(rc, rc);
7424	return rc;
7425	}
7426
7427
7428	/**
7429	* Queue up a VMWRITE by using the VMCS write cache.
7430	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
7431	*
7432	* @param pVCpu The cross context virtual CPU structure.
7433	* @param idxField The VMCS field encoding.
7434	* @param u64Val 16, 32 or 64-bit value.
7435	*/
7436	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7437	{
7438	AssertPtr(pVCpu);
7439	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7440
7441	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
7442	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
7443
7444	/* Make sure there are no duplicates. */
7445	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
7446	{
7447	if (pCache->Write.aField[i] == idxField)
7448	{
7449	pCache->Write.aFieldVal[i] = u64Val;
7450	return VINF_SUCCESS;
7451	}
7452	}
7453
7454	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
7455	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
7456	pCache->Write.cValidEntries++;
7457	return VINF_SUCCESS;
7458	}
7459	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
7460
7461
7462	/**
7463	* Sets up the usage of TSC-offsetting and updates the VMCS.
7464	*
7465	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
7466	* VMX-preemption timer.
7467	*
7468	* @returns VBox status code.
7469	* @param pVCpu The cross context virtual CPU structure.
7470	* @param pVmxTransient The VMX-transient structure.
7471	*
7472	* @remarks No-long-jump zone!!!
7473	*/
7474	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
7475	{
7476	bool fOffsettedTsc;
7477	bool fParavirtTsc;
7478	uint64_t uTscOffset;
7479	PVM pVM = pVCpu->CTX_SUFF(pVM);
7480	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7481
7482	if (pVM->hm.s.vmx.fUsePreemptTimer)
7483	{
7484	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
7485
7486	/* Make sure the returned values have sane upper and lower boundaries. */
7487	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
7488	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
7489	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
7490	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
7491
7492	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
7493	* preemption timers here. We probably need to clamp the preemption timer,
7494	* after converting the timer value to the host. */
7495	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7496	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7497	AssertRC(rc);
7498	}
7499	else
7500	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7501
7502	if (fParavirtTsc)
7503	{
7504	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7505	information before every VM-entry, hence disable it for performance sake. */
7506	#if 0
7507	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7508	AssertRC(rc);
7509	#endif
7510	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7511	}
7512
7513	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
7514	if ( fOffsettedTsc
7515	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7516	{
7517	if (pVmxTransient->fIsNestedGuest)
7518	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7519	if (pVmcsInfo->u64TscOffset != uTscOffset)
7520	{
7521	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
7522	AssertRC(rc);
7523	pVmcsInfo->u64TscOffset = uTscOffset;
7524	}
7525
7526	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
7527	{
7528	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
7529	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7530	AssertRC(rc);
7531	pVmcsInfo->u32ProcCtls = uProcCtls;
7532	}
7533	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
7534	}
7535	else
7536	{
7537	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7538	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
7539	{
7540	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
7541	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7542	AssertRC(rc);
7543	pVmcsInfo->u32ProcCtls = uProcCtls;
7544	}
7545	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
7546	}
7547	}
7548
7549
7550	/**
7551	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7552	* VM-exit interruption info type.
7553	*
7554	* @returns The IEM exception flags.
7555	* @param uVector The event vector.
7556	* @param uVmxEventType The VMX event type.
7557	*
7558	* @remarks This function currently only constructs flags required for
7559	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7560	* and CR2 aspects of an exception are not included).
7561	*/
7562	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7563	{
7564	uint32_t fIemXcptFlags;
7565	switch (uVmxEventType)
7566	{
7567	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7568	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7569	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7570	break;
7571
7572	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7573	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7574	break;
7575
7576	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7577	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7578	break;
7579
7580	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7581	{
7582	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7583	if (uVector == X86_XCPT_BP)
7584	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7585	else if (uVector == X86_XCPT_OF)
7586	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7587	else
7588	{
7589	fIemXcptFlags = 0;
7590	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7591	}
7592	break;
7593	}
7594
7595	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7596	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7597	break;
7598
7599	default:
7600	fIemXcptFlags = 0;
7601	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7602	break;
7603	}
7604	return fIemXcptFlags;
7605	}
7606
7607
7608	/**
7609	* Sets an event as a pending event to be injected into the guest.
7610	*
7611	* @param pVCpu The cross context virtual CPU structure.
7612	* @param u32IntInfo The VM-entry interruption-information field.
7613	* @param cbInstr The VM-entry instruction length in bytes (for software
7614	* interrupts, exceptions and privileged software
7615	* exceptions).
7616	* @param u32ErrCode The VM-entry exception error code.
7617	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7618	* page-fault.
7619	*/
7620	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7621	RTGCUINTPTR GCPtrFaultAddress)
7622	{
7623	Assert(!pVCpu->hm.s.Event.fPending);
7624	pVCpu->hm.s.Event.fPending = true;
7625	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7626	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7627	pVCpu->hm.s.Event.cbInstr = cbInstr;
7628	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7629	}
7630
7631
7632	/**
7633	* Sets an external interrupt as pending-for-injection into the VM.
7634	*
7635	* @param pVCpu The cross context virtual CPU structure.
7636	* @param u8Interrupt The external interrupt vector.
7637	*/
7638	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7639	{
7640	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7641	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7642	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7643	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7644	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7645	}
7646
7647
7648	/**
7649	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7650	*
7651	* @param pVCpu The cross context virtual CPU structure.
7652	*/
7653	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7654	{
7655	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7656	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7657	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7658	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7659	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7660	}
7661
7662
7663	/**
7664	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7665	*
7666	* @param pVCpu The cross context virtual CPU structure.
7667	*/
7668	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7669	{
7670	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7672	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7673	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7674	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7675	}
7676
7677
7678	/**
7679	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7680	*
7681	* @param pVCpu The cross context virtual CPU structure.
7682	*/
7683	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7684	{
7685	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7686	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7687	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7688	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7689	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7690	}
7691
7692
7693	/**
7694	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7695	*
7696	* @param pVCpu The cross context virtual CPU structure.
7697	*/
7698	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7699	{
7700	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7701	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7702	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7703	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7704	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7705	}
7706
7707
7708	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7709	/**
7710	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7711	*
7712	* @param pVCpu The cross context virtual CPU structure.
7713	* @param u32ErrCode The error code for the general-protection exception.
7714	*/
7715	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7716	{
7717	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7718	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7719	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7720	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7721	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7722	}
7723
7724
7725	/**
7726	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7727	*
7728	* @param pVCpu The cross context virtual CPU structure.
7729	* @param u32ErrCode The error code for the stack exception.
7730	*/
7731	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7732	{
7733	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7734	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7735	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7736	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7737	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7738	}
7739	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7740
7741
7742	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7743	{
7744	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7745
7746	/*
7747	* If VT-x marks the segment as unusable, most other bits remain undefined:
7748	* - For CS the L, D and G bits have meaning.
7749	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7750	* - For the remaining data segments no bits are defined.
7751	*
7752	* The present bit and the unusable bit has been observed to be set at the
7753	* same time (the selector was supposed to be invalid as we started executing
7754	* a V8086 interrupt in ring-0).
7755	*
7756	* What should be important for the rest of the VBox code, is that the P bit is
7757	* cleared. Some of the other VBox code recognizes the unusable bit, but
7758	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7759	* safe side here, we'll strip off P and other bits we don't care about. If
7760	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7761	*
7762	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7763	*/
7764	#ifdef VBOX_STRICT
7765	uint32_t const uAttr = pSelReg->Attr.u;
7766	#endif
7767
7768	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7769	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7770	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7771
7772	#ifdef VBOX_STRICT
7773	VMMRZCallRing3Disable(pVCpu);
7774	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7775	# ifdef DEBUG_bird
7776	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7777	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7778	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7779	# endif
7780	VMMRZCallRing3Enable(pVCpu);
7781	NOREF(uAttr);
7782	#endif
7783	RT_NOREF2(pVCpu, idxSel);
7784	}
7785
7786
7787	/**
7788	* Imports a guest segment register from the current VMCS into the guest-CPU
7789	* context.
7790	*
7791	* @returns VBox status code.
7792	* @param pVCpu The cross context virtual CPU structure.
7793	* @param iSegReg The segment register number (X86_SREG_XXX).
7794	*
7795	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7796	* do not log!
7797	*/
7798	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7799	{
7800	Assert(iSegReg < X86_SREG_COUNT);
7801
7802	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7803	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7804	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7805	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7806	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7807	#else
7808	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7809	#endif
7810	uint64_t u64Base;
7811	uint32_t u32Sel, u32Limit, u32Attr;
7812	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7813	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7814	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7815	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7816	if (RT_SUCCESS(rc))
7817	{
7818	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7819	pSelReg->Sel = u32Sel;
7820	pSelReg->ValidSel = u32Sel;
7821	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7822	pSelReg->u32Limit = u32Limit;
7823	pSelReg->u64Base = u64Base;
7824	pSelReg->Attr.u = u32Attr;
7825	if (u32Attr & X86DESCATTR_UNUSABLE)
7826	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7827	}
7828	return rc;
7829	}
7830
7831
7832	/**
7833	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7834	*
7835	* @returns VBox status code.
7836	* @param pVCpu The cross context virtual CPU structure.
7837	*
7838	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7839	* do not log!
7840	*/
7841	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7842	{
7843	uint64_t u64Base;
7844	uint32_t u32Sel, u32Limit, u32Attr;
7845	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7846	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7847	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7848	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7849	if (RT_SUCCESS(rc))
7850	{
7851	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7852	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7853	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7854	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7855	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7856	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7857	if (u32Attr & X86DESCATTR_UNUSABLE)
7858	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7859	}
7860	return rc;
7861	}
7862
7863
7864	/**
7865	* Imports the guest TR from the current VMCS into the guest-CPU context.
7866	*
7867	* @returns VBox status code.
7868	* @param pVCpu The cross context virtual CPU structure.
7869	*
7870	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7871	* do not log!
7872	*/
7873	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7874	{
7875	uint32_t u32Sel, u32Limit, u32Attr;
7876	uint64_t u64Base;
7877	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7878	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7879	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7880	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7881	AssertRCReturn(rc, rc);
7882
7883	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7884	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7885	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7886	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7887	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7888	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7889	/* TR is the only selector that can never be unusable. */
7890	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7891	return VINF_SUCCESS;
7892	}
7893
7894
7895	/**
7896	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7897	*
7898	* @returns VBox status code.
7899	* @param pVCpu The cross context virtual CPU structure.
7900	*
7901	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7902	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7903	* instead!!!
7904	*/
7905	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7906	{
7907	uint64_t u64Val;
7908	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7909	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7910	{
7911	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7912	if (RT_SUCCESS(rc))
7913	{
7914	pCtx->rip = u64Val;
7915	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7916	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7917	}
7918	return rc;
7919	}
7920	return VINF_SUCCESS;
7921	}
7922
7923
7924	/**
7925	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7926	*
7927	* @returns VBox status code.
7928	* @param pVCpu The cross context virtual CPU structure.
7929	* @param pVmcsInfo The VMCS info. object.
7930	*
7931	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7932	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7933	* instead!!!
7934	*/
7935	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7936	{
7937	uint32_t u32Val;
7938	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7939	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7940	{
7941	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7942	if (RT_SUCCESS(rc))
7943	{
7944	pCtx->eflags.u32 = u32Val;
7945
7946	/* Restore eflags for real-on-v86-mode hack. */
7947	if (pVmcsInfo->RealMode.fRealOnV86Active)
7948	{
7949	pCtx->eflags.Bits.u1VM = 0;
7950	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7951	}
7952	}
7953	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7954	return rc;
7955	}
7956	return VINF_SUCCESS;
7957	}
7958
7959
7960	/**
7961	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7962	* context.
7963	*
7964	* @returns VBox status code.
7965	* @param pVCpu The cross context virtual CPU structure.
7966	* @param pVmcsInfo The VMCS info. object.
7967	*
7968	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7969	* do not log!
7970	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7971	* instead!!!
7972	*/
7973	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7974	{
7975	uint32_t u32Val;
7976	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7977	if (RT_SUCCESS(rc))
7978	{
7979	if (!u32Val)
7980	{
7981	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7982	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7983
7984	CPUMSetGuestNmiBlocking(pVCpu, false);
7985	}
7986	else
7987	{
7988	/*
7989	* We must import RIP here to set our EM interrupt-inhibited state.
7990	* We also import RFLAGS as our code that evaluates pending interrupts
7991	* before VM-entry requires it.
7992	*/
7993	rc = hmR0VmxImportGuestRip(pVCpu);
7994	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7995	if (RT_SUCCESS(rc))
7996	{
7997	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7998	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7999	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
8000	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
8001
8002	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8003	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
8004	}
8005	}
8006	}
8007	return rc;
8008	}
8009
8010
8011	/**
8012	* Worker for VMXR0ImportStateOnDemand.
8013	*
8014	* @returns VBox status code.
8015	* @param pVCpu The cross context virtual CPU structure.
8016	* @param pVmcsInfo The VMCS info. object.
8017	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8018	*/
8019	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
8020	{
8021	#define VMXLOCAL_BREAK_RC(a_rc) \
8022	if (RT_SUCCESS(a_rc)) \
8023	{ } \
8024	else \
8025	break
8026
8027	int rc = VINF_SUCCESS;
8028	PVM pVM = pVCpu->CTX_SUFF(pVM);
8029	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8030	uint64_t u64Val;
8031	uint32_t u32Val;
8032
8033	/*
8034	* Note! This is hack to workaround a mysterious BSOD observed with release builds
8035	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
8036	* neither are other host platforms.
8037	*
8038	* Committing this temporarily as it prevents BSOD.
8039	*
8040	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
8041	*/
8042	#ifdef RT_OS_WINDOWS
8043	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
8044	return VERR_HM_IPE_1;
8045	#endif
8046
8047	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
8048
8049	/*
8050	* We disable interrupts to make the updating of the state and in particular
8051	* the fExtrn modification atomic wrt to preemption hooks.
8052	*/
8053	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
8054
8055	fWhat &= pCtx->fExtrn;
8056	if (fWhat)
8057	{
8058	do
8059	{
8060	if (fWhat & CPUMCTX_EXTRN_RIP)
8061	{
8062	rc = hmR0VmxImportGuestRip(pVCpu);
8063	VMXLOCAL_BREAK_RC(rc);
8064	}
8065
8066	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
8067	{
8068	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8069	VMXLOCAL_BREAK_RC(rc);
8070	}
8071
8072	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
8073	{
8074	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
8075	VMXLOCAL_BREAK_RC(rc);
8076	}
8077
8078	if (fWhat & CPUMCTX_EXTRN_RSP)
8079	{
8080	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
8081	VMXLOCAL_BREAK_RC(rc);
8082	pCtx->rsp = u64Val;
8083	}
8084
8085	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
8086	{
8087	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
8088	if (fWhat & CPUMCTX_EXTRN_CS)
8089	{
8090	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
8091	rc \|= hmR0VmxImportGuestRip(pVCpu);
8092	if (fRealOnV86Active)
8093	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
8094	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
8095	}
8096	if (fWhat & CPUMCTX_EXTRN_SS)
8097	{
8098	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
8099	if (fRealOnV86Active)
8100	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
8101	}
8102	if (fWhat & CPUMCTX_EXTRN_DS)
8103	{
8104	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
8105	if (fRealOnV86Active)
8106	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
8107	}
8108	if (fWhat & CPUMCTX_EXTRN_ES)
8109	{
8110	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
8111	if (fRealOnV86Active)
8112	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
8113	}
8114	if (fWhat & CPUMCTX_EXTRN_FS)
8115	{
8116	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
8117	if (fRealOnV86Active)
8118	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
8119	}
8120	if (fWhat & CPUMCTX_EXTRN_GS)
8121	{
8122	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
8123	if (fRealOnV86Active)
8124	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
8125	}
8126	VMXLOCAL_BREAK_RC(rc);
8127	}
8128
8129	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
8130	{
8131	if (fWhat & CPUMCTX_EXTRN_LDTR)
8132	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
8133
8134	if (fWhat & CPUMCTX_EXTRN_GDTR)
8135	{
8136	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8137	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8138	pCtx->gdtr.pGdt = u64Val;
8139	pCtx->gdtr.cbGdt = u32Val;
8140	}
8141
8142	/* Guest IDTR. */
8143	if (fWhat & CPUMCTX_EXTRN_IDTR)
8144	{
8145	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8146	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8147	pCtx->idtr.pIdt = u64Val;
8148	pCtx->idtr.cbIdt = u32Val;
8149	}
8150
8151	/* Guest TR. */
8152	if (fWhat & CPUMCTX_EXTRN_TR)
8153	{
8154	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
8155	don't need to import that one. */
8156	if (!pVmcsInfo->RealMode.fRealOnV86Active)
8157	rc \|= hmR0VmxImportGuestTr(pVCpu);
8158	}
8159	VMXLOCAL_BREAK_RC(rc);
8160	}
8161
8162	if (fWhat & CPUMCTX_EXTRN_DR7)
8163	{
8164	if (!pVCpu->hm.s.fUsingHyperDR7)
8165	{
8166	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
8167	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
8168	VMXLOCAL_BREAK_RC(rc);
8169	pCtx->dr[7] = u32Val;
8170	}
8171	}
8172
8173	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
8174	{
8175	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
8176	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
8177	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
8178	pCtx->SysEnter.cs = u32Val;
8179	VMXLOCAL_BREAK_RC(rc);
8180	}
8181
8182	#if HC_ARCH_BITS == 64
8183	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
8184	{
8185	if ( pVM->hm.s.fAllow64BitGuests
8186	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8187	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
8188	}
8189
8190	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
8191	{
8192	if ( pVM->hm.s.fAllow64BitGuests
8193	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8194	{
8195	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
8196	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
8197	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
8198	}
8199	}
8200	#endif
8201
8202	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
8203	#if HC_ARCH_BITS == 32
8204	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
8205	#endif
8206	)
8207	{
8208	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
8209	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
8210	Assert(pMsrs);
8211	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
8212	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
8213	for (uint32_t i = 0; i < cMsrs; i++)
8214	{
8215	uint32_t const idMsr = pMsrs[i].u32Msr;
8216	switch (idMsr)
8217	{
8218	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
8219	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
8220	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
8221	#if HC_ARCH_BITS == 32
8222	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
8223	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
8224	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
8225	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
8226	#endif
8227	default:
8228	{
8229	pCtx->fExtrn = 0;
8230	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
8231	ASMSetFlags(fEFlags);
8232	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
8233	return VERR_HM_UNEXPECTED_LD_ST_MSR;
8234	}
8235	}
8236	}
8237	}
8238
8239	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
8240	{
8241	uint64_t u64Shadow;
8242	if (fWhat & CPUMCTX_EXTRN_CR0)
8243	{
8244	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8245	* remove when we drop 32-bit host w/ 64-bit host support, see
8246	* @bugref{9180#c39}. */
8247	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8248	#if HC_ARCH_BITS == 32
8249	uint32_t u32Shadow;
8250	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
8251	u64Shadow = u32Shadow;
8252	#else
8253	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
8254	#endif
8255	VMXLOCAL_BREAK_RC(rc);
8256	u64Val = u32Val;
8257	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
8258	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
8259	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8260	/*
8261	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
8262	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
8263	*/
8264	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8265	{
8266	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
8267	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
8268	}
8269	#endif
8270	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
8271	CPUMSetGuestCR0(pVCpu, u64Val);
8272	VMMRZCallRing3Enable(pVCpu);
8273	}
8274
8275	if (fWhat & CPUMCTX_EXTRN_CR4)
8276	{
8277	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8278	* remove when we drop 32-bit host w/ 64-bit host support, see
8279	* @bugref{9180#c39}. */
8280	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8281	#if HC_ARCH_BITS == 32
8282	uint32_t u32Shadow;
8283	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
8284	u64Shadow = u32Shadow;
8285	#else
8286	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
8287	#endif
8288	VMXLOCAL_BREAK_RC(rc);
8289	u64Val = u32Val;
8290	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
8291	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
8292	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8293	/*
8294	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
8295	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
8296	*/
8297	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8298	{
8299	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
8300	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
8301	}
8302	#endif
8303	pCtx->cr4 = u64Val;
8304	}
8305
8306	if (fWhat & CPUMCTX_EXTRN_CR3)
8307	{
8308	/* CR0.PG bit changes are always intercepted, so it's up to date. */
8309	if ( pVM->hm.s.vmx.fUnrestrictedGuest
8310	\|\| ( pVM->hm.s.fNestedPaging
8311	&& CPUMIsGuestPagingEnabledEx(pCtx)))
8312	{
8313	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
8314	VMXLOCAL_BREAK_RC(rc);
8315	if (pCtx->cr3 != u64Val)
8316	{
8317	pCtx->cr3 = u64Val;
8318	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
8319	}
8320
8321	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
8322	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
8323	if (CPUMIsGuestInPAEModeEx(pCtx))
8324	{
8325	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
8326	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
8327	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
8328	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
8329	VMXLOCAL_BREAK_RC(rc);
8330	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
8331	}
8332	}
8333	}
8334	}
8335
8336	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8337	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
8338	{
8339	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
8340	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
8341	{
8342	Assert(CPUMIsGuestInVmxRootMode(pCtx));
8343	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
8344	VMXLOCAL_BREAK_RC(rc);
8345	}
8346
8347	# if 0
8348	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
8349	* VM-exit handlers as parameters. We would handle it differently when using
8350	* the fast path. */
8351	/*
8352	* The hardware virtualization state currently consists of VMCS fields that may be
8353	* modified by execution of the nested-guest (that are not part of the general
8354	* guest state) and is visible to guest software. Hence, it is technically part of
8355	* the guest-CPU state when executing a nested-guest.
8356	*/
8357	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8358	{
8359	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
8360	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
8361	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
8362	VMXLOCAL_BREAK_RC(rc);
8363
8364	/*
8365	* VM-entry can fail due to invalid-guest state, machine-check events and
8366	* MSR loading failures. Other than VM-exit reason and Exit qualification
8367	* all other VMCS fields are left unmodified on VM-entry failure.
8368	*
8369	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
8370	*/
8371	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
8372	if (!fEntryFailed)
8373	{
8374	/*
8375	* Some notes on VMCS fields that may need importing when the fast path
8376	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
8377	*
8378	* Requires fixing up when using hardware-assisted VMX:
8379	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
8380	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
8381	* - IDT-vectoring info: Think about this.
8382	* - IDT-vectoring error code: Think about this.
8383	*
8384	* Emulated:
8385	* - Guest-interruptiblity state: Derived from FFs and RIP.
8386	* - Guest pending debug exceptions: Derived from DR6.
8387	* - Guest activity state: Emulated from EM state.
8388	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
8389	* - Entry-interrupt info: Emulated, cleared to 0.
8390	*/
8391	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
8392	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
8393	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
8394	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
8395	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
8396	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
8397	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
8398	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
8399	/** @todo NSTVMX: Save and adjust preemption timer value. */
8400	}
8401
8402	VMXLOCAL_BREAK_RC(rc);
8403	}
8404	# endif
8405	}
8406	#endif
8407	} while (0);
8408
8409	if (RT_SUCCESS(rc))
8410	{
8411	/* Update fExtrn. */
8412	pCtx->fExtrn &= ~fWhat;
8413
8414	/* If everything has been imported, clear the HM keeper bit. */
8415	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
8416	{
8417	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
8418	Assert(!pCtx->fExtrn);
8419	}
8420	}
8421	}
8422	else
8423	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
8424
8425	ASMSetFlags(fEFlags);
8426
8427	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
8428
8429	if (RT_SUCCESS(rc))
8430	{ /* likely */ }
8431	else
8432	return rc;
8433
8434	/*
8435	* Honor any pending CR3 updates.
8436	*
8437	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
8438	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
8439	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
8440	*
8441	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
8442	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
8443	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
8444	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
8445	*
8446	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
8447	*/
8448	if (VMMRZCallRing3IsEnabled(pVCpu))
8449	{
8450	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8451	{
8452	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8453	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8454	}
8455
8456	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8457	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8458
8459	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8460	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8461	}
8462
8463	return VINF_SUCCESS;
8464	#undef VMXLOCAL_BREAK_RC
8465	}
8466
8467
8468	/**
8469	* Saves the guest state from the VMCS into the guest-CPU context.
8470	*
8471	* @returns VBox status code.
8472	* @param pVCpu The cross context virtual CPU structure.
8473	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8474	*/
8475	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
8476	{
8477	AssertPtr(pVCpu);
8478	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8479	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
8480	}
8481
8482
8483	/**
8484	* Check per-VM and per-VCPU force flag actions that require us to go back to
8485	* ring-3 for one reason or another.
8486	*
8487	* @returns Strict VBox status code (i.e. informational status codes too)
8488	* @retval VINF_SUCCESS if we don't have any actions that require going back to
8489	* ring-3.
8490	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
8491	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
8492	* interrupts)
8493	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
8494	* all EMTs to be in ring-3.
8495	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
8496	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
8497	* to the EM loop.
8498	*
8499	* @param pVCpu The cross context virtual CPU structure.
8500	* @param fStepping Whether we are single-stepping the guest using the
8501	* hypervisor debugger.
8502	*/
8503	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
8504	{
8505	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8506
8507	/*
8508	* Update pending interrupts into the APIC's IRR.
8509	*/
8510	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8511	APICUpdatePendingInterrupts(pVCpu);
8512
8513	/*
8514	* Anything pending? Should be more likely than not if we're doing a good job.
8515	*/
8516	PVM pVM = pVCpu->CTX_SUFF(pVM);
8517	if ( !fStepping
8518	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
8519	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
8520	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
8521	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
8522	return VINF_SUCCESS;
8523
8524	/* Pending PGM C3 sync. */
8525	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
8526	{
8527	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8528	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
8529	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8530	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8531	if (rcStrict2 != VINF_SUCCESS)
8532	{
8533	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8534	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8535	return rcStrict2;
8536	}
8537	}
8538
8539	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8540	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8541	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8542	{
8543	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8544	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8545	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8546	return rc2;
8547	}
8548
8549	/* Pending VM request packets, such as hardware interrupts. */
8550	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8551	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8552	{
8553	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
8554	Log4Func(("Pending VM request forcing us back to ring-3\n"));
8555	return VINF_EM_PENDING_REQUEST;
8556	}
8557
8558	/* Pending PGM pool flushes. */
8559	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8560	{
8561	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
8562	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8563	return VINF_PGM_POOL_FLUSH_PENDING;
8564	}
8565
8566	/* Pending DMA requests. */
8567	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8568	{
8569	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
8570	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8571	return VINF_EM_RAW_TO_R3;
8572	}
8573
8574	return VINF_SUCCESS;
8575	}
8576
8577
8578	/**
8579	* Converts any TRPM trap into a pending HM event. This is typically used when
8580	* entering from ring-3 (not longjmp returns).
8581	*
8582	* @param pVCpu The cross context virtual CPU structure.
8583	*/
8584	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8585	{
8586	Assert(TRPMHasTrap(pVCpu));
8587	Assert(!pVCpu->hm.s.Event.fPending);
8588
8589	uint8_t uVector;
8590	TRPMEVENT enmTrpmEvent;
8591	RTGCUINT uErrCode;
8592	RTGCUINTPTR GCPtrFaultAddress;
8593	uint8_t cbInstr;
8594
8595	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8596	AssertRC(rc);
8597
8598	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
8599	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
8600	if (enmTrpmEvent == TRPM_TRAP)
8601	{
8602	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
8603	* generated using INT1 (ICEBP). */
8604	switch (uVector)
8605	{
8606	case X86_XCPT_NMI:
8607	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8608	break;
8609
8610	case X86_XCPT_BP:
8611	case X86_XCPT_OF:
8612	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8613	break;
8614
8615	case X86_XCPT_PF:
8616	case X86_XCPT_DF:
8617	case X86_XCPT_TS:
8618	case X86_XCPT_NP:
8619	case X86_XCPT_SS:
8620	case X86_XCPT_GP:
8621	case X86_XCPT_AC:
8622	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
8623	RT_FALL_THRU();
8624	default:
8625	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8626	break;
8627	}
8628	}
8629	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
8630	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8631	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
8632	{
8633	switch (uVector)
8634	{
8635	case X86_XCPT_BP:
8636	case X86_XCPT_OF:
8637	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8638	break;
8639
8640	default:
8641	Assert(uVector == X86_XCPT_DB);
8642	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8643	break;
8644	}
8645	}
8646	else
8647	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
8648
8649	rc = TRPMResetTrap(pVCpu);
8650	AssertRC(rc);
8651	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8652	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8653
8654	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8655	}
8656
8657
8658	/**
8659	* Converts the pending HM event into a TRPM trap.
8660	*
8661	* @param pVCpu The cross context virtual CPU structure.
8662	*/
8663	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8664	{
8665	Assert(pVCpu->hm.s.Event.fPending);
8666
8667	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8668	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
8669	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
8670	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
8671
8672	/* If a trap was already pending, we did something wrong! */
8673	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8674
8675	/** @todo Use HMVmxEventToTrpmEventType() later. */
8676	TRPMEVENT enmTrapType;
8677	switch (uVectorType)
8678	{
8679	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
8680	enmTrapType = TRPM_HARDWARE_INT;
8681	break;
8682
8683	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
8684	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
8685	enmTrapType = TRPM_TRAP;
8686	break;
8687
8688	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
8689	Assert(uVector == X86_XCPT_DB);
8690	enmTrapType = TRPM_SOFTWARE_INT;
8691	break;
8692
8693	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
8694	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8695	enmTrapType = TRPM_SOFTWARE_INT;
8696	break;
8697
8698	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8699	enmTrapType = TRPM_SOFTWARE_INT;
8700	break;
8701
8702	default:
8703	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
8704	enmTrapType = TRPM_32BIT_HACK;
8705	break;
8706	}
8707
8708	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8709
8710	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8711	AssertRC(rc);
8712
8713	if (fErrorCodeValid)
8714	TRPMSetErrorCode(pVCpu, uErrorCode);
8715
8716	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8717	&& uVector == X86_XCPT_PF)
8718	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8719	else if (enmTrapType == TRPM_SOFTWARE_INT)
8720	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8721
8722	/* We're now done converting the pending event. */
8723	pVCpu->hm.s.Event.fPending = false;
8724	}
8725
8726
8727	/**
8728	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8729	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8730	*
8731	* @param pVCpu The cross context virtual CPU structure.
8732	* @param pVmcsInfo The VMCS info. object.
8733	*/
8734	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8735	{
8736	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8737	{
8738	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8739	{
8740	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8741	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8742	AssertRC(rc);
8743	}
8744	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8745	}
8746
8747
8748	/**
8749	* Clears the interrupt-window exiting control in the VMCS.
8750	*
8751	* @param pVmcsInfo The VMCS info. object.
8752	*/
8753	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8754	{
8755	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8756	{
8757	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8758	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8759	}
8760	return VINF_SUCCESS;
8761	}
8762
8763
8764	/**
8765	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8766	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8767	*
8768	* @param pVCpu The cross context virtual CPU structure.
8769	* @param pVmcsInfo The VMCS info. object.
8770	*/
8771	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8772	{
8773	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8774	{
8775	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8776	{
8777	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8778	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8779	AssertRC(rc);
8780	Log4Func(("Setup NMI-window exiting\n"));
8781	}
8782	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8783	}
8784
8785
8786	/**
8787	* Clears the NMI-window exiting control in the VMCS.
8788	*
8789	* @param pVmcsInfo The VMCS info. object.
8790	*/
8791	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8792	{
8793	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8794	{
8795	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8796	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8797	}
8798	return VINF_SUCCESS;
8799	}
8800
8801
8802	/**
8803	* Does the necessary state syncing before returning to ring-3 for any reason
8804	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8805	*
8806	* @returns VBox status code.
8807	* @param pVCpu The cross context virtual CPU structure.
8808	* @param fImportState Whether to import the guest state from the VMCS back
8809	* to the guest-CPU context.
8810	*
8811	* @remarks No-long-jmp zone!!!
8812	*/
8813	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8814	{
8815	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8816	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8817
8818	RTCPUID const idCpu = RTMpCpuId();
8819	Log4Func(("HostCpuId=%u\n", idCpu));
8820
8821	/*
8822	* !!! IMPORTANT !!!
8823	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8824	*/
8825
8826	/* Save the guest state if necessary. */
8827	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8828	if (fImportState)
8829	{
8830	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8831	AssertRCReturn(rc, rc);
8832	}
8833
8834	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8835	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8836	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8837
8838	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8839	#ifdef VBOX_STRICT
8840	if (CPUMIsHyperDebugStateActive(pVCpu))
8841	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8842	#endif
8843	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8844	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8845	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8846
8847	#if HC_ARCH_BITS == 64
8848	/* Restore host-state bits that VT-x only restores partially. */
8849	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8850	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8851	{
8852	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8853	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8854	}
8855	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8856	#endif
8857
8858	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8859	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8860	{
8861	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8862	if (!fImportState)
8863	{
8864	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8865	AssertRCReturn(rc, rc);
8866	}
8867	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8868	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8869	}
8870	else
8871	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8872
8873	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8874	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8875
8876	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8877	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8878	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8879	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8880	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8881	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8882	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8883	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8884	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8885	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8886
8887	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8888
8889	/** @todo This partially defeats the purpose of having preemption hooks.
8890	* The problem is, deregistering the hooks should be moved to a place that
8891	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8892	* context.
8893	*/
8894	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8895	AssertRCReturn(rc, rc);
8896
8897	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8898	/*
8899	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8900	* clear a shadow VMCS before allowing that VMCS to become active on another
8901	* logical processor. We may or may not be importing guest state which clears
8902	* it, so cover for it here.
8903	*
8904	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8905	*/
8906	if ( pVmcsInfo->pvShadowVmcs
8907	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8908	{
8909	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8910	AssertRCReturn(rc, rc);
8911	}
8912
8913	/*
8914	* Flag that we need to re-import the host state if we switch to this VMCS before
8915	* executing guest or nested-guest code.
8916	*/
8917	pVmcsInfo->idHostCpu = NIL_RTCPUID;
8918	#endif
8919
8920	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8921	NOREF(idCpu);
8922	return VINF_SUCCESS;
8923	}
8924
8925
8926	/**
8927	* Leaves the VT-x session.
8928	*
8929	* @returns VBox status code.
8930	* @param pVCpu The cross context virtual CPU structure.
8931	*
8932	* @remarks No-long-jmp zone!!!
8933	*/
8934	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8935	{
8936	HM_DISABLE_PREEMPT(pVCpu);
8937	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8938	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8939	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8940
8941	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8942	and done this from the VMXR0ThreadCtxCallback(). */
8943	if (!pVCpu->hm.s.fLeaveDone)
8944	{
8945	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8946	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8947	pVCpu->hm.s.fLeaveDone = true;
8948	}
8949	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8950
8951	/*
8952	* !!! IMPORTANT !!!
8953	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8954	*/
8955
8956	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8957	/** @todo Deregistering here means we need to VMCLEAR always
8958	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8959	* for calling VMMR0ThreadCtxHookDisable here! */
8960	VMMR0ThreadCtxHookDisable(pVCpu);
8961
8962	/* Leave HM context. This takes care of local init (term). */
8963	int rc = HMR0LeaveCpu(pVCpu);
8964
8965	HM_RESTORE_PREEMPT();
8966	return rc;
8967	}
8968
8969
8970	/**
8971	* Does the necessary state syncing before doing a longjmp to ring-3.
8972	*
8973	* @returns VBox status code.
8974	* @param pVCpu The cross context virtual CPU structure.
8975	*
8976	* @remarks No-long-jmp zone!!!
8977	*/
8978	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8979	{
8980	return hmR0VmxLeaveSession(pVCpu);
8981	}
8982
8983
8984	/**
8985	* Take necessary actions before going back to ring-3.
8986	*
8987	* An action requires us to go back to ring-3. This function does the necessary
8988	* steps before we can safely return to ring-3. This is not the same as longjmps
8989	* to ring-3, this is voluntary and prepares the guest so it may continue
8990	* executing outside HM (recompiler/IEM).
8991	*
8992	* @returns VBox status code.
8993	* @param pVCpu The cross context virtual CPU structure.
8994	* @param rcExit The reason for exiting to ring-3. Can be
8995	* VINF_VMM_UNKNOWN_RING3_CALL.
8996	*/
8997	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8998	{
8999	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9000
9001	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9002	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
9003	{
9004	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
9005	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
9006	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
9007	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
9008	}
9009
9010	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
9011	VMMRZCallRing3Disable(pVCpu);
9012	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
9013
9014	/*
9015	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
9016	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
9017	*
9018	* This is because execution may continue from ring-3 and we would need to inject
9019	* the event from there (hence place it back in TRPM).
9020	*/
9021	if (pVCpu->hm.s.Event.fPending)
9022	{
9023	hmR0VmxPendingEventToTrpmTrap(pVCpu);
9024	Assert(!pVCpu->hm.s.Event.fPending);
9025
9026	/* Clear the events from the VMCS. */
9027	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
9028	AssertRCReturn(rc, rc);
9029	}
9030	#ifdef VBOX_STRICT
9031	else
9032	{
9033	/*
9034	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
9035	* This can be pretty hard to debug otherwise, interrupts might get injected twice
9036	* occasionally, see @bugref{9180#c42}.
9037	*
9038	* However, if the VM-entry failed, any VM entry-interruption info. field would
9039	* be left unmodified as the event would not have been injected to the guest. In
9040	* such cases, don't assert, we're not going to continue guest execution anyway.
9041	*/
9042	uint32_t uExitReason;
9043	uint32_t uEntryIntInfo;
9044	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9045	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
9046	AssertRC(rc);
9047	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
9048	}
9049	#endif
9050
9051	/*
9052	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
9053	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
9054	* (e.g. TPR below threshold).
9055	*/
9056	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9057	{
9058	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
9059	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
9060	AssertRCReturn(rc, rc);
9061	}
9062
9063	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
9064	and if we're injecting an event we should have a TRPM trap pending. */
9065	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
9066	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
9067	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
9068	#endif
9069
9070	/* Save guest state and restore host state bits. */
9071	int rc = hmR0VmxLeaveSession(pVCpu);
9072	AssertRCReturn(rc, rc);
9073	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
9074
9075	/* Thread-context hooks are unregistered at this point!!! */
9076
9077	/* Sync recompiler state. */
9078	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
9079	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
9080	\| CPUM_CHANGED_LDTR
9081	\| CPUM_CHANGED_GDTR
9082	\| CPUM_CHANGED_IDTR
9083	\| CPUM_CHANGED_TR
9084	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
9085	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
9086	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
9087	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
9088
9089	Assert(!pVCpu->hm.s.fClearTrapFlag);
9090
9091	/* Update the exit-to-ring 3 reason. */
9092	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
9093
9094	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
9095	if ( rcExit != VINF_EM_RAW_INTERRUPT
9096	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9097	{
9098	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
9099	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9100	}
9101
9102	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
9103
9104	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
9105	VMMRZCallRing3RemoveNotification(pVCpu);
9106	VMMRZCallRing3Enable(pVCpu);
9107
9108	return rc;
9109	}
9110
9111
9112	/**
9113	* VMMRZCallRing3() callback wrapper which saves the guest state before we
9114	* longjump to ring-3 and possibly get preempted.
9115	*
9116	* @returns VBox status code.
9117	* @param pVCpu The cross context virtual CPU structure.
9118	* @param enmOperation The operation causing the ring-3 longjump.
9119	* @param pvUser User argument, currently unused, NULL.
9120	*/
9121	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
9122	{
9123	RT_NOREF(pvUser);
9124	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
9125	{
9126	/*
9127	* !!! IMPORTANT !!!
9128	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
9129	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
9130	*/
9131	VMMRZCallRing3RemoveNotification(pVCpu);
9132	VMMRZCallRing3Disable(pVCpu);
9133	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
9134	RTThreadPreemptDisable(&PreemptState);
9135
9136	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9137	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9138	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
9139	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
9140
9141	#if HC_ARCH_BITS == 64
9142	/* Restore host-state bits that VT-x only restores partially. */
9143	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
9144	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
9145	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
9146	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
9147	#endif
9148
9149	/* Restore the lazy host MSRs as we're leaving VT-x context. */
9150	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
9151	hmR0VmxLazyRestoreHostMsrs(pVCpu);
9152
9153	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
9154	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
9155	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
9156
9157	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
9158	cleared as part of importing the guest state above. */
9159	hmR0VmxClearVmcs(pVmcsInfo);
9160
9161	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
9162	VMMR0ThreadCtxHookDisable(pVCpu);
9163	HMR0LeaveCpu(pVCpu);
9164	RTThreadPreemptRestore(&PreemptState);
9165	return VINF_SUCCESS;
9166	}
9167
9168	Assert(pVCpu);
9169	Assert(pvUser);
9170	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9171	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9172
9173	VMMRZCallRing3Disable(pVCpu);
9174	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9175
9176	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
9177
9178	int rc = hmR0VmxLongJmpToRing3(pVCpu);
9179	AssertRCReturn(rc, rc);
9180
9181	VMMRZCallRing3Enable(pVCpu);
9182	return VINF_SUCCESS;
9183	}
9184
9185
9186	/**
9187	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
9188	* stack.
9189	*
9190	* @returns Strict VBox status code (i.e. informational status codes too).
9191	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
9192	* @param pVCpu The cross context virtual CPU structure.
9193	* @param uValue The value to push to the guest stack.
9194	*/
9195	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
9196	{
9197	/*
9198	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
9199	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
9200	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
9201	*/
9202	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9203	if (pCtx->sp == 1)
9204	return VINF_EM_RESET;
9205	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
9206	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
9207	AssertRC(rc);
9208	return rc;
9209	}
9210
9211
9212	/**
9213	* Injects an event into the guest upon VM-entry by updating the relevant fields
9214	* in the VM-entry area in the VMCS.
9215	*
9216	* @returns Strict VBox status code (i.e. informational status codes too).
9217	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
9218	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
9219	*
9220	* @param pVCpu The cross context virtual CPU structure.
9221	* @param pVmxTransient The VMX-transient structure.
9222	* @param pEvent The event being injected.
9223	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
9224	* will be updated if necessary. This cannot not be NULL.
9225	* @param fStepping Whether we're single-stepping guest execution and should
9226	* return VINF_EM_DBG_STEPPED if the event is injected
9227	* directly (registers modified by us, not by hardware on
9228	* VM-entry).
9229	*/
9230	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
9231	uint32_t *pfIntrState)
9232	{
9233	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
9234	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
9235	Assert(pfIntrState);
9236
9237	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9238	uint32_t u32IntInfo = pEvent->u64IntInfo;
9239	uint32_t const u32ErrCode = pEvent->u32ErrCode;
9240	uint32_t const cbInstr = pEvent->cbInstr;
9241	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
9242	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
9243	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
9244
9245	#ifdef VBOX_STRICT
9246	/*
9247	* Validate the error-code-valid bit for hardware exceptions.
9248	* No error codes for exceptions in real-mode.
9249	*
9250	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9251	*/
9252	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9253	&& !CPUMIsGuestInRealModeEx(pCtx))
9254	{
9255	switch (uVector)
9256	{
9257	case X86_XCPT_PF:
9258	case X86_XCPT_DF:
9259	case X86_XCPT_TS:
9260	case X86_XCPT_NP:
9261	case X86_XCPT_SS:
9262	case X86_XCPT_GP:
9263	case X86_XCPT_AC:
9264	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
9265	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
9266	RT_FALL_THRU();
9267	default:
9268	break;
9269	}
9270	}
9271
9272	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
9273	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
9274	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9275	#endif
9276
9277	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
9278
9279	/*
9280	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
9281	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
9282	* interrupt handler in the (real-mode) guest.
9283	*
9284	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
9285	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
9286	*/
9287	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
9288	{
9289	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
9290	{
9291	/*
9292	* For CPUs with unrestricted guest execution enabled and with the guest
9293	* in real-mode, we must not set the deliver-error-code bit.
9294	*
9295	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
9296	*/
9297	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
9298	}
9299	else
9300	{
9301	PVM pVM = pVCpu->CTX_SUFF(pVM);
9302	Assert(PDMVmmDevHeapIsEnabled(pVM));
9303	Assert(pVM->hm.s.vmx.pRealModeTSS);
9304	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
9305
9306	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
9307	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9308	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
9309	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
9310	AssertRCReturn(rc2, rc2);
9311
9312	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
9313	size_t const cbIdtEntry = sizeof(X86IDTR16);
9314	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
9315	{
9316	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
9317	if (uVector == X86_XCPT_DF)
9318	return VINF_EM_RESET;
9319
9320	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
9321	No error codes for exceptions in real-mode. */
9322	if (uVector == X86_XCPT_GP)
9323	{
9324	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
9325	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9326	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9327	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9328	HMEVENT EventXcptDf;
9329	RT_ZERO(EventXcptDf);
9330	EventXcptDf.u64IntInfo = uXcptDfInfo;
9331	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
9332	}
9333
9334	/*
9335	* If we're injecting an event with no valid IDT entry, inject a #GP.
9336	* No error codes for exceptions in real-mode.
9337	*
9338	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9339	*/
9340	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
9341	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9342	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9343	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9344	HMEVENT EventXcptGp;
9345	RT_ZERO(EventXcptGp);
9346	EventXcptGp.u64IntInfo = uXcptGpInfo;
9347	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
9348	}
9349
9350	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
9351	uint16_t uGuestIp = pCtx->ip;
9352	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
9353	{
9354	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9355	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
9356	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9357	}
9358	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
9359	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9360
9361	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
9362	X86IDTR16 IdtEntry;
9363	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
9364	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
9365	AssertRCReturn(rc2, rc2);
9366
9367	/* Construct the stack frame for the interrupt/exception handler. */
9368	VBOXSTRICTRC rcStrict;
9369	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
9370	if (rcStrict == VINF_SUCCESS)
9371	{
9372	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
9373	if (rcStrict == VINF_SUCCESS)
9374	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
9375	}
9376
9377	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
9378	if (rcStrict == VINF_SUCCESS)
9379	{
9380	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
9381	pCtx->rip = IdtEntry.offSel;
9382	pCtx->cs.Sel = IdtEntry.uSel;
9383	pCtx->cs.ValidSel = IdtEntry.uSel;
9384	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
9385	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
9386	&& uVector == X86_XCPT_PF)
9387	pCtx->cr2 = GCPtrFault;
9388
9389	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
9390	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
9391	\| HM_CHANGED_GUEST_RSP);
9392
9393	/*
9394	* If we delivered a hardware exception (other than an NMI) and if there was
9395	* block-by-STI in effect, we should clear it.
9396	*/
9397	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9398	{
9399	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
9400	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
9401	Log4Func(("Clearing inhibition due to STI\n"));
9402	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
9403	}
9404
9405	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
9406	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
9407
9408	/*
9409	* The event has been truly dispatched to the guest. Mark it as no longer pending so
9410	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
9411	*/
9412	pVCpu->hm.s.Event.fPending = false;
9413
9414	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
9415	if (fStepping)
9416	rcStrict = VINF_EM_DBG_STEPPED;
9417	}
9418	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9419	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9420	return rcStrict;
9421	}
9422	}
9423
9424	/*
9425	* Validate.
9426	*/
9427	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
9428	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
9429
9430	/*
9431	* Inject the event into the VMCS.
9432	*/
9433	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
9434	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
9435	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
9436	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
9437	AssertRCReturn(rc, rc);
9438
9439	/*
9440	* Update guest CR2 if this is a page-fault.
9441	*/
9442	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9443	&& uVector == X86_XCPT_PF)
9444	pCtx->cr2 = GCPtrFault;
9445
9446	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
9447	return VINF_SUCCESS;
9448	}
9449
9450
9451	/**
9452	* Evaluates the event to be delivered to the guest and sets it as the pending
9453	* event.
9454	*
9455	* @returns Strict VBox status code (i.e. informational status codes too).
9456	* @param pVCpu The cross context virtual CPU structure.
9457	* @param pVmxTransient The VMX-transient structure.
9458	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
9459	*/
9460	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
9461	{
9462	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9463	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9464	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
9465
9466	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
9467	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
9468	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9469	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9470	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9471
9472	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9473	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9474	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9475	Assert(!TRPMHasTrap(pVCpu));
9476	Assert(pfIntrState);
9477
9478	*pfIntrState = fIntrState;
9479
9480	/*
9481	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
9482	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
9483	*/
9484	/** @todo SMI. SMIs take priority over NMIs. */
9485	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
9486	{
9487	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
9488	if ( !pVCpu->hm.s.Event.fPending
9489	&& !fBlockNmi
9490	&& !fBlockSti
9491	&& !fBlockMovSS)
9492	{
9493	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9494	if ( fIsNestedGuest
9495	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
9496	return IEMExecVmxVmexitXcptNmi(pVCpu);
9497	#endif
9498	hmR0VmxSetPendingXcptNmi(pVCpu);
9499	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
9500	Log4Func(("Pending NMI\n"));
9501	}
9502	else if (!fIsNestedGuest)
9503	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
9504	}
9505	/*
9506	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
9507	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
9508	*/
9509	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
9510	&& !pVCpu->hm.s.fSingleInstruction)
9511	{
9512	Assert(!DBGFIsStepping(pVCpu));
9513	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9514	AssertRCReturn(rc, rc);
9515	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
9516	if ( !pVCpu->hm.s.Event.fPending
9517	&& !fBlockInt
9518	&& !fBlockSti
9519	&& !fBlockMovSS)
9520	{
9521	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9522	if ( fIsNestedGuest
9523	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
9524	{
9525	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
9526	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9527	return rcStrict;
9528	}
9529	#endif
9530	uint8_t u8Interrupt;
9531	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
9532	if (RT_SUCCESS(rc))
9533	{
9534	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9535	if ( fIsNestedGuest
9536	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
9537	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
9538	{
9539	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
9540	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9541	return rcStrict;
9542	}
9543	#endif
9544	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
9545	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
9546	}
9547	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
9548	{
9549	if ( !fIsNestedGuest
9550	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
9551	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
9552	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
9553
9554	/*
9555	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
9556	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
9557	* need to re-set this force-flag here.
9558	*/
9559	}
9560	else
9561	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
9562	}
9563	else if (!fIsNestedGuest)
9564	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
9565	}
9566
9567	return VINF_SUCCESS;
9568	}
9569
9570
9571	/**
9572	* Injects any pending events into the guest if the guest is in a state to
9573	* receive them.
9574	*
9575	* @returns Strict VBox status code (i.e. informational status codes too).
9576	* @param pVCpu The cross context virtual CPU structure.
9577	* @param pVmxTransient The VMX-transient structure.
9578	* @param fIntrState The VT-x guest-interruptibility state.
9579	* @param fStepping Whether we are single-stepping the guest using the
9580	* hypervisor debugger and should return
9581	* VINF_EM_DBG_STEPPED if the event was dispatched
9582	* directly.
9583	*/
9584	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9585	{
9586	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9587	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9588
9589	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9590	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9591
9592	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9593	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9594	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9595	Assert(!TRPMHasTrap(pVCpu));
9596
9597	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9598	if (pVCpu->hm.s.Event.fPending)
9599	{
9600	/*
9601	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9602	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9603	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9604	*
9605	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9606	*/
9607	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9608	#ifdef VBOX_STRICT
9609	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9610	{
9611	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9612	Assert(!fBlockInt);
9613	Assert(!fBlockSti);
9614	Assert(!fBlockMovSS);
9615	}
9616	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9617	{
9618	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9619	Assert(!fBlockSti);
9620	Assert(!fBlockMovSS);
9621	Assert(!fBlockNmi);
9622	}
9623	#endif
9624	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9625	uIntType));
9626
9627	/*
9628	* Inject the event and get any changes to the guest-interruptibility state.
9629	*
9630	* The guest-interruptibility state may need to be updated if we inject the event
9631	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9632	*/
9633	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9634	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9635
9636	/*
9637	* If we are executing a nested-guest make sure that we should intercept subsequent
9638	* events. The one we are injecting might be part of VM-entry.
9639	*/
9640	if (pVmxTransient->fIsNestedGuest)
9641	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
9642
9643	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9644	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9645	else
9646	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9647	}
9648
9649	/*
9650	* Update the guest-interruptibility state.
9651	*
9652	* This is required for the real-on-v86 software interrupt injection case above, as well as
9653	* updates to the guest state from ring-3 or IEM/REM.
9654	*/
9655	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9656	AssertRCReturn(rc, rc);
9657
9658	/*
9659	* There's no need to clear the VM-entry interruption-information field here if we're not
9660	* injecting anything. VT-x clears the valid bit on every VM-exit.
9661	*
9662	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9663	*/
9664
9665	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9666	NOREF(fBlockMovSS); NOREF(fBlockSti);
9667	return rcStrict;
9668	}
9669
9670
9671	/**
9672	* Enters the VT-x session.
9673	*
9674	* @returns VBox status code.
9675	* @param pVCpu The cross context virtual CPU structure.
9676	*/
9677	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9678	{
9679	AssertPtr(pVCpu);
9680	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9681	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9682
9683	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9684	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9685	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9686
9687	#ifdef VBOX_STRICT
9688	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9689	RTCCUINTREG uHostCr4 = ASMGetCR4();
9690	if (!(uHostCr4 & X86_CR4_VMXE))
9691	{
9692	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9693	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9694	}
9695	#endif
9696
9697	/*
9698	* Load the appropriate VMCS as the current and active one.
9699	*/
9700	PVMXVMCSINFO pVmcsInfo;
9701	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9702	if (!fInNestedGuestMode)
9703	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9704	else
9705	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9706	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9707	if (RT_SUCCESS(rc))
9708	{
9709	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9710	pVCpu->hm.s.fLeaveDone = false;
9711	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9712
9713	/*
9714	* Do the EMT scheduled L1D flush here if needed.
9715	*/
9716	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9717	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9718	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9719	hmR0MdsClear();
9720	}
9721	return rc;
9722	}
9723
9724
9725	/**
9726	* The thread-context callback (only on platforms which support it).
9727	*
9728	* @param enmEvent The thread-context event.
9729	* @param pVCpu The cross context virtual CPU structure.
9730	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9731	* @thread EMT(pVCpu)
9732	*/
9733	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9734	{
9735	AssertPtr(pVCpu);
9736	RT_NOREF1(fGlobalInit);
9737
9738	switch (enmEvent)
9739	{
9740	case RTTHREADCTXEVENT_OUT:
9741	{
9742	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9743	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9744	VMCPU_ASSERT_EMT(pVCpu);
9745
9746	/* No longjmps (logger flushes, locks) in this fragile context. */
9747	VMMRZCallRing3Disable(pVCpu);
9748	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9749
9750	/* Restore host-state (FPU, debug etc.) */
9751	if (!pVCpu->hm.s.fLeaveDone)
9752	{
9753	/*
9754	* Do -not- import the guest-state here as we might already be in the middle of importing
9755	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9756	*/
9757	hmR0VmxLeave(pVCpu, false /* fImportState */);
9758	pVCpu->hm.s.fLeaveDone = true;
9759	}
9760
9761	/* Leave HM context, takes care of local init (term). */
9762	int rc = HMR0LeaveCpu(pVCpu);
9763	AssertRC(rc);
9764
9765	/* Restore longjmp state. */
9766	VMMRZCallRing3Enable(pVCpu);
9767	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9768	break;
9769	}
9770
9771	case RTTHREADCTXEVENT_IN:
9772	{
9773	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9774	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9775	VMCPU_ASSERT_EMT(pVCpu);
9776
9777	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9778	VMMRZCallRing3Disable(pVCpu);
9779	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9780
9781	/* Initialize the bare minimum state required for HM. This takes care of
9782	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9783	int rc = hmR0EnterCpu(pVCpu);
9784	AssertRC(rc);
9785	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9786	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9787
9788	/* Load the active VMCS as the current one. */
9789	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9790	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9791	AssertRC(rc);
9792	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9793	pVCpu->hm.s.fLeaveDone = false;
9794
9795	/* Do the EMT scheduled L1D flush if needed. */
9796	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9797	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9798
9799	/* Restore longjmp state. */
9800	VMMRZCallRing3Enable(pVCpu);
9801	break;
9802	}
9803
9804	default:
9805	break;
9806	}
9807	}
9808
9809
9810	/**
9811	* Exports the host state into the VMCS host-state area.
9812	* Sets up the VM-exit MSR-load area.
9813	*
9814	* The CPU state will be loaded from these fields on every successful VM-exit.
9815	*
9816	* @returns VBox status code.
9817	* @param pVCpu The cross context virtual CPU structure.
9818	*
9819	* @remarks No-long-jump zone!!!
9820	*/
9821	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9822	{
9823	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9824
9825	int rc = VINF_SUCCESS;
9826	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9827	{
9828	rc = hmR0VmxExportHostControlRegs();
9829	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9830
9831	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9832	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9833
9834	rc = hmR0VmxExportHostMsrs(pVCpu);
9835	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9836
9837	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9838	}
9839	return rc;
9840	}
9841
9842
9843	/**
9844	* Saves the host state in the VMCS host-state.
9845	*
9846	* @returns VBox status code.
9847	* @param pVCpu The cross context virtual CPU structure.
9848	*
9849	* @remarks No-long-jump zone!!!
9850	*/
9851	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9852	{
9853	AssertPtr(pVCpu);
9854	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9855
9856	/*
9857	* Export the host state here while entering HM context.
9858	* When thread-context hooks are used, we might get preempted and have to re-save the host
9859	* state but most of the time we won't be, so do it here before we disable interrupts.
9860	*/
9861	return hmR0VmxExportHostState(pVCpu);
9862	}
9863
9864
9865	/**
9866	* Exports the guest state into the VMCS guest-state area.
9867	*
9868	* The will typically be done before VM-entry when the guest-CPU state and the
9869	* VMCS state may potentially be out of sync.
9870	*
9871	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9872	* VM-entry controls.
9873	* Sets up the appropriate VMX non-root function to execute guest code based on
9874	* the guest CPU mode.
9875	*
9876	* @returns VBox strict status code.
9877	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9878	* without unrestricted guest execution and the VMMDev is not presently
9879	* mapped (e.g. EFI32).
9880	*
9881	* @param pVCpu The cross context virtual CPU structure.
9882	* @param pVmxTransient The VMX-transient structure.
9883	*
9884	* @remarks No-long-jump zone!!!
9885	*/
9886	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9887	{
9888	AssertPtr(pVCpu);
9889	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9890	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9891
9892	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9893
9894	/*
9895	* Determine real-on-v86 mode.
9896	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9897	*/
9898	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9899	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9900	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9901	pVmcsInfo->RealMode. fRealOnV86Active = false;
9902	else
9903	{
9904	Assert(!pVmxTransient->fIsNestedGuest);
9905	pVmcsInfo->RealMode.fRealOnV86Active = true;
9906	}
9907
9908	/*
9909	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9910	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9911	*/
9912	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9913	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9914	* be a need to evaluate this everytime since I'm pretty sure we intercept
9915	* all guest paging mode changes. */
9916	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9917	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9918
9919	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9920	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9921
9922	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9923	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9924
9925	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9926	if (rcStrict == VINF_SUCCESS)
9927	{ /* likely */ }
9928	else
9929	{
9930	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9931	return rcStrict;
9932	}
9933
9934	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9935	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9936
9937	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9938	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9939
9940	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9941	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9942
9943	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9944	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9945
9946	rc = hmR0VmxExportGuestRip(pVCpu);
9947	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9948	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9949	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9950
9951	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9952	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9953
9954	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9955	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9956	\| HM_CHANGED_GUEST_CR2
9957	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9958	\| HM_CHANGED_GUEST_X87
9959	\| HM_CHANGED_GUEST_SSE_AVX
9960	\| HM_CHANGED_GUEST_OTHER_XSAVE
9961	\| HM_CHANGED_GUEST_XCRx
9962	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9963	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9964	\| HM_CHANGED_GUEST_TSC_AUX
9965	\| HM_CHANGED_GUEST_OTHER_MSRS
9966	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9967
9968	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9969	return rc;
9970	}
9971
9972
9973	/**
9974	* Exports the state shared between the host and guest into the VMCS.
9975	*
9976	* @param pVCpu The cross context virtual CPU structure.
9977	* @param pVmxTransient The VMX-transient structure.
9978	*
9979	* @remarks No-long-jump zone!!!
9980	*/
9981	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9982	{
9983	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9984	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9985
9986	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9987	{
9988	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9989	AssertRC(rc);
9990	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9991
9992	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9993	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9994	{
9995	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9996	AssertRC(rc);
9997	}
9998	}
9999
10000	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
10001	{
10002	hmR0VmxLazyLoadGuestMsrs(pVCpu);
10003	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
10004	}
10005
10006	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
10007	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10008	}
10009
10010
10011	/**
10012	* Worker for loading the guest-state bits in the inner VT-x execution loop.
10013	*
10014	* @returns Strict VBox status code (i.e. informational status codes too).
10015	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
10016	* without unrestricted guest execution and the VMMDev is not presently
10017	* mapped (e.g. EFI32).
10018	*
10019	* @param pVCpu The cross context virtual CPU structure.
10020	* @param pVmxTransient The VMX-transient structure.
10021	*
10022	* @remarks No-long-jump zone!!!
10023	*/
10024	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10025	{
10026	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10027	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10028	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10029
10030	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
10031	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
10032	#endif
10033
10034	/*
10035	* For many exits it's only RIP that changes and hence try to export it first
10036	* without going through a lot of change flag checks.
10037	*/
10038	VBOXSTRICTRC rcStrict;
10039	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
10040	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
10041	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
10042	{
10043	rcStrict = hmR0VmxExportGuestRip(pVCpu);
10044	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10045	{ /* likely */}
10046	else
10047	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
10048	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
10049	}
10050	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
10051	{
10052	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
10053	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10054	{ /* likely */}
10055	else
10056	{
10057	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
10058	VBOXSTRICTRC_VAL(rcStrict)));
10059	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10060	return rcStrict;
10061	}
10062	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
10063	}
10064	else
10065	rcStrict = VINF_SUCCESS;
10066
10067	#ifdef VBOX_STRICT
10068	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
10069	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
10070	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
10071	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
10072	("fCtxChanged=%#RX64\n", fCtxChanged));
10073	#endif
10074	return rcStrict;
10075	}
10076
10077
10078	/**
10079	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10080	* and update error record fields accordingly.
10081	*
10082	* @returns VMX_IGS_* error codes.
10083	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10084	* wrong with the guest state.
10085	*
10086	* @param pVCpu The cross context virtual CPU structure.
10087	* @param pVmcsInfo The VMCS info. object.
10088	*
10089	* @remarks This function assumes our cache of the VMCS controls
10090	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10091	*/
10092	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10093	{
10094	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10095	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10096	uError = (err); \
10097	break; \
10098	} else do { } while (0)
10099
10100	int rc;
10101	PVM pVM = pVCpu->CTX_SUFF(pVM);
10102	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10103	uint32_t uError = VMX_IGS_ERROR;
10104	uint32_t u32Val;
10105	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10106
10107	do
10108	{
10109	/*
10110	* CR0.
10111	*/
10112	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10113	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10114	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10115	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
10116	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10117	if (fUnrestrictedGuest)
10118	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10119
10120	uint32_t u32GuestCr0;
10121	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10122	AssertRCBreak(rc);
10123	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10124	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10125	if ( !fUnrestrictedGuest
10126	&& (u32GuestCr0 & X86_CR0_PG)
10127	&& !(u32GuestCr0 & X86_CR0_PE))
10128	{
10129	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10130	}
10131
10132	/*
10133	* CR4.
10134	*/
10135	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10136	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10137	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10138
10139	uint32_t u32GuestCr4;
10140	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10141	AssertRCBreak(rc);
10142	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10143	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10144
10145	/*
10146	* IA32_DEBUGCTL MSR.
10147	*/
10148	uint64_t u64Val;
10149	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10150	AssertRCBreak(rc);
10151	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10152	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10153	{
10154	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10155	}
10156	uint64_t u64DebugCtlMsr = u64Val;
10157
10158	#ifdef VBOX_STRICT
10159	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10160	AssertRCBreak(rc);
10161	Assert(u32Val == pVmcsInfo->u32EntryCtls);
10162	#endif
10163	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10164
10165	/*
10166	* RIP and RFLAGS.
10167	*/
10168	uint32_t u32Eflags;
10169	#if HC_ARCH_BITS == 64
10170	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10171	AssertRCBreak(rc);
10172	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10173	if ( !fLongModeGuest
10174	\|\| !pCtx->cs.Attr.n.u1Long)
10175	{
10176	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10177	}
10178	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10179	* must be identical if the "IA-32e mode guest" VM-entry
10180	* control is 1 and CS.L is 1. No check applies if the
10181	* CPU supports 64 linear-address bits. */
10182
10183	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10184	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10185	AssertRCBreak(rc);
10186	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10187	VMX_IGS_RFLAGS_RESERVED);
10188	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10189	u32Eflags = u64Val;
10190	#else
10191	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10192	AssertRCBreak(rc);
10193	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10194	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10195	#endif
10196
10197	if ( fLongModeGuest
10198	\|\| ( fUnrestrictedGuest
10199	&& !(u32GuestCr0 & X86_CR0_PE)))
10200	{
10201	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10202	}
10203
10204	uint32_t u32EntryInfo;
10205	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10206	AssertRCBreak(rc);
10207	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10208	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10209	{
10210	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10211	}
10212
10213	/*
10214	* 64-bit checks.
10215	*/
10216	#if HC_ARCH_BITS == 64
10217	if (fLongModeGuest)
10218	{
10219	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10220	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10221	}
10222
10223	if ( !fLongModeGuest
10224	&& (u32GuestCr4 & X86_CR4_PCIDE))
10225	{
10226	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10227	}
10228
10229	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10230	* 51:32 beyond the processor's physical-address width are 0. */
10231
10232	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10233	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10234	{
10235	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10236	}
10237
10238	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10239	AssertRCBreak(rc);
10240	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10241
10242	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10243	AssertRCBreak(rc);
10244	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10245	#endif
10246
10247	/*
10248	* PERF_GLOBAL MSR.
10249	*/
10250	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10251	{
10252	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10253	AssertRCBreak(rc);
10254	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10255	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10256	}
10257
10258	/*
10259	* PAT MSR.
10260	*/
10261	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10262	{
10263	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10264	AssertRCBreak(rc);
10265	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10266	for (unsigned i = 0; i < 8; i++)
10267	{
10268	uint8_t u8Val = (u64Val & 0xff);
10269	if ( u8Val != 0 /* UC */
10270	&& u8Val != 1 /* WC */
10271	&& u8Val != 4 /* WT */
10272	&& u8Val != 5 /* WP */
10273	&& u8Val != 6 /* WB */
10274	&& u8Val != 7 /* UC- */)
10275	{
10276	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10277	}
10278	u64Val >>= 8;
10279	}
10280	}
10281
10282	/*
10283	* EFER MSR.
10284	*/
10285	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10286	{
10287	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10288	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10289	AssertRCBreak(rc);
10290	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10291	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10292	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
10293	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10294	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10295	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10296	* iemVmxVmentryCheckGuestState(). */
10297	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10298	\|\| !(u32GuestCr0 & X86_CR0_PG)
10299	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10300	VMX_IGS_EFER_LMA_LME_MISMATCH);
10301	}
10302
10303	/*
10304	* Segment registers.
10305	*/
10306	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10307	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10308	if (!(u32Eflags & X86_EFL_VM))
10309	{
10310	/* CS */
10311	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10312	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10313	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10314	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10315	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10316	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10317	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10318	/* CS cannot be loaded with NULL in protected mode. */
10319	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10320	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10321	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10322	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10323	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10324	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10325	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10326	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10327	else
10328	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10329
10330	/* SS */
10331	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10332	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10333	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10334	if ( !(pCtx->cr0 & X86_CR0_PE)
10335	\|\| pCtx->cs.Attr.n.u4Type == 3)
10336	{
10337	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10338	}
10339	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10340	{
10341	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10342	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10343	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10344	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10345	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10346	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10347	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10348	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10349	}
10350
10351	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
10352	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10353	{
10354	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10355	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10356	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10357	\|\| pCtx->ds.Attr.n.u4Type > 11
10358	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10359	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10360	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10361	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10362	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10363	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10364	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10365	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10366	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10367	}
10368	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10369	{
10370	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10371	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10372	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10373	\|\| pCtx->es.Attr.n.u4Type > 11
10374	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10375	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10376	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10377	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10378	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10379	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10380	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10381	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10382	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10383	}
10384	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10385	{
10386	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10387	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10388	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10389	\|\| pCtx->fs.Attr.n.u4Type > 11
10390	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10391	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10392	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10393	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10394	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10395	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10396	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10397	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10398	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10399	}
10400	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10401	{
10402	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10403	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10404	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10405	\|\| pCtx->gs.Attr.n.u4Type > 11
10406	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10407	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10408	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10409	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10410	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10411	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10412	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10413	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10414	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10415	}
10416	/* 64-bit capable CPUs. */
10417	#if HC_ARCH_BITS == 64
10418	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10419	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10420	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10421	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10422	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10423	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10424	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10425	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10426	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10427	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10428	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10429	#endif
10430	}
10431	else
10432	{
10433	/* V86 mode checks. */
10434	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10435	if (pVmcsInfo->RealMode.fRealOnV86Active)
10436	{
10437	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10438	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10439	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10440	}
10441	else
10442	{
10443	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10444	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10445	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10446	}
10447
10448	/* CS */
10449	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10450	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10451	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10452	/* SS */
10453	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10454	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10455	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10456	/* DS */
10457	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10458	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10459	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10460	/* ES */
10461	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10462	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10463	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10464	/* FS */
10465	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10466	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10467	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10468	/* GS */
10469	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10470	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10471	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10472	/* 64-bit capable CPUs. */
10473	#if HC_ARCH_BITS == 64
10474	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10475	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10476	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10477	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10478	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10479	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10480	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10481	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10482	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10483	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10484	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10485	#endif
10486	}
10487
10488	/*
10489	* TR.
10490	*/
10491	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10492	/* 64-bit capable CPUs. */
10493	#if HC_ARCH_BITS == 64
10494	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10495	#endif
10496	if (fLongModeGuest)
10497	{
10498	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10499	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10500	}
10501	else
10502	{
10503	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10504	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10505	VMX_IGS_TR_ATTR_TYPE_INVALID);
10506	}
10507	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10508	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10509	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10510	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10511	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10512	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10513	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10514	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10515
10516	/*
10517	* GDTR and IDTR.
10518	*/
10519	#if HC_ARCH_BITS == 64
10520	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10521	AssertRCBreak(rc);
10522	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10523
10524	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10525	AssertRCBreak(rc);
10526	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10527	#endif
10528
10529	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10530	AssertRCBreak(rc);
10531	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10532
10533	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10534	AssertRCBreak(rc);
10535	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10536
10537	/*
10538	* Guest Non-Register State.
10539	*/
10540	/* Activity State. */
10541	uint32_t u32ActivityState;
10542	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10543	AssertRCBreak(rc);
10544	HMVMX_CHECK_BREAK( !u32ActivityState
10545	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10546	VMX_IGS_ACTIVITY_STATE_INVALID);
10547	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10548	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10549	uint32_t u32IntrState;
10550	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10551	AssertRCBreak(rc);
10552	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10553	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10554	{
10555	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10556	}
10557
10558	/** @todo Activity state and injecting interrupts. Left as a todo since we
10559	* currently don't use activity states but ACTIVE. */
10560
10561	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10562	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10563
10564	/* Guest interruptibility-state. */
10565	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10566	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10567	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10568	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10569	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10570	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10571	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10572	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10573	{
10574	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10575	{
10576	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10577	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10578	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10579	}
10580	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10581	{
10582	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10583	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10584	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10585	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10586	}
10587	}
10588	/** @todo Assumes the processor is not in SMM. */
10589	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10590	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10591	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10592	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10593	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10594	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10595	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10596	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10597	{
10598	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10599	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10600	}
10601
10602	/* Pending debug exceptions. */
10603	#if HC_ARCH_BITS == 64
10604	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10605	AssertRCBreak(rc);
10606	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10607	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10608	u32Val = u64Val; /* For pending debug exceptions checks below. */
10609	#else
10610	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10611	AssertRCBreak(rc);
10612	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10613	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10614	#endif
10615
10616	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10617	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10618	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10619	{
10620	if ( (u32Eflags & X86_EFL_TF)
10621	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10622	{
10623	/* Bit 14 is PendingDebug.BS. */
10624	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10625	}
10626	if ( !(u32Eflags & X86_EFL_TF)
10627	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10628	{
10629	/* Bit 14 is PendingDebug.BS. */
10630	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10631	}
10632	}
10633
10634	/* VMCS link pointer. */
10635	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10636	AssertRCBreak(rc);
10637	if (u64Val != UINT64_C(0xffffffffffffffff))
10638	{
10639	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10640	/** @todo Bits beyond the processor's physical-address width MBZ. */
10641	/** @todo SMM checks. */
10642	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10643	Assert(pVmcsInfo->pvShadowVmcs);
10644	VMXVMCSREVID VmcsRevId;
10645	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10646	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10647	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10648	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10649	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10650	}
10651
10652	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10653	* not using nested paging? */
10654	if ( pVM->hm.s.fNestedPaging
10655	&& !fLongModeGuest
10656	&& CPUMIsGuestInPAEModeEx(pCtx))
10657	{
10658	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10659	AssertRCBreak(rc);
10660	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10661
10662	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10663	AssertRCBreak(rc);
10664	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10665
10666	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10667	AssertRCBreak(rc);
10668	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10669
10670	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10671	AssertRCBreak(rc);
10672	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10673	}
10674
10675	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10676	if (uError == VMX_IGS_ERROR)
10677	uError = VMX_IGS_REASON_NOT_FOUND;
10678	} while (0);
10679
10680	pVCpu->hm.s.u32HMError = uError;
10681	return uError;
10682
10683	#undef HMVMX_ERROR_BREAK
10684	#undef HMVMX_CHECK_BREAK
10685	}
10686
10687
10688	/**
10689	* Map the APIC-access page for virtualizing APIC accesses.
10690	*
10691	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10692	* this not done as part of exporting guest state, see @bugref{8721}.
10693	*
10694	* @returns VBox status code.
10695	* @param pVCpu The cross context virtual CPU structure.
10696	*/
10697	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10698	{
10699	PVM pVM = pVCpu->CTX_SUFF(pVM);
10700	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10701
10702	Assert(PDMHasApic(pVM));
10703	Assert(u64MsrApicBase);
10704
10705	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10706	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10707
10708	/* Unalias the existing mapping. */
10709	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10710	AssertRCReturn(rc, rc);
10711
10712	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10713	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10714	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10715	AssertRCReturn(rc, rc);
10716
10717	/* Update the per-VCPU cache of the APIC base MSR. */
10718	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10719	return VINF_SUCCESS;
10720	}
10721
10722
10723	/**
10724	* Wrapper for dispatching host NMIs.
10725	*
10726	* @returns VBox status code.
10727	* @param pVCpu The cross context virtual CPU structure.
10728	*/
10729	static int hmR0VmxExitHostNmi(PVMCPU pVCpu)
10730	{
10731	VMXDispatchHostNmi();
10732	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10733	return VINF_SUCCESS;
10734	}
10735
10736
10737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10738	/**
10739	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10740	* nested-guest using hardware-assisted VMX.
10741	*
10742	* @param pVCpu The cross context virtual CPU structure.
10743	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10744	* @param pVmcsInfoGst The guest VMCS info. object.
10745	*/
10746	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10747	{
10748	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10749	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10750	Assert(pu64MsrBitmap);
10751
10752	/*
10753	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10754	* MSR that is intercepted by the guest is also intercepted while executing the
10755	* nested-guest using hardware-assisted VMX.
10756	*
10757	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10758	* nested-guest VM-exit even if the outer guest is not intercepting some
10759	* MSRs. We cannot assume the caller has initialized the nested-guest
10760	* MSR bitmap in this case.
10761	*
10762	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10763	* each VM-entry, hence initializing it once per-VM while setting up the
10764	* nested-guest VMCS is not sufficient.
10765	*/
10766	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10767	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10768	{
10769	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10770	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10771	Assert(pu64MsrBitmapNstGst);
10772	Assert(pu64MsrBitmapGst);
10773
10774	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10775	for (uint32_t i = 0; i < cFrags; i++)
10776	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10777	}
10778	else
10779	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10780	}
10781
10782
10783	/**
10784	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10785	* hardware-assisted VMX execution of the nested-guest.
10786	*
10787	* For a guest, we don't modify these controls once we set up the VMCS and hence
10788	* this function is never called.
10789	*
10790	* For nested-guests since the guest hypervisor provides these controls on every
10791	* nested-guest VM-entry and could potentially change them everytime we need to
10792	* merge them before every nested-guest VM-entry.
10793	*
10794	* @returns VBox status code.
10795	* @param pVCpu The cross context virtual CPU structure.
10796	*/
10797	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10798	{
10799	PVM pVM = pVCpu->CTX_SUFF(pVM);
10800	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10801	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10802	Assert(pVmcsNstGst);
10803
10804	/*
10805	* Merge the controls with the requirements of the guest VMCS.
10806	*
10807	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10808	* VMCS with the features supported by the physical CPU as it's already done by the
10809	* VMLAUNCH/VMRESUME instruction emulation.
10810	*
10811	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10812	* derived from the VMX features supported by the physical CPU.
10813	*/
10814
10815	/* Pin-based VM-execution controls. */
10816	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10817
10818	/* Processor-based VM-execution controls. */
10819	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10820	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10821	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10822	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10823	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10824
10825	/* Secondary processor-based VM-execution controls. */
10826	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10827	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10828	\| VMX_PROC_CTLS2_INVPCID
10829	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10830	\| VMX_PROC_CTLS2_RDTSCP
10831	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10832	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10833	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10834	\| VMX_PROC_CTLS2_VMFUNC));
10835
10836	/*
10837	* VM-entry controls:
10838	* These controls contains state that depends on the nested-guest state (primarily
10839	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10840	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10841	* properly continue executing the nested-guest if the EFER MSR changes but does not
10842	* cause a nested-guest VM-exits.
10843	*
10844	* VM-exit controls:
10845	* These controls specify the host state on return. We cannot use the controls from
10846	* the guest hypervisor state as is as it would contain the guest state rather than
10847	* the host state. Since the host state is subject to change (e.g. preemption, trips
10848	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10849	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10850	*
10851	* VM-entry MSR-load:
10852	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10853	* context by the VMLAUNCH/VMRESUME instruction emulation.
10854	*
10855	* VM-exit MSR-store:
10856	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10857	* back into the VM-exit MSR-store area.
10858	*
10859	* VM-exit MSR-load areas:
10860	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10861	* can entirely ignore what the guest hypervisor wants to load here.
10862	*/
10863
10864	/*
10865	* Exception bitmap.
10866	*
10867	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10868	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10869	* code more flexible if intercepting exceptions become more dynamic in the future we do
10870	* it as part of exporting the nested-guest state.
10871	*/
10872	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10873
10874	/*
10875	* CR0/CR4 guest/host mask.
10876	*
10877	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10878	* cause VM-exits, so we need to merge them here.
10879	*/
10880	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10881	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10882
10883	/*
10884	* Page-fault error-code mask and match.
10885	*
10886	* Although we require unrestricted guest execution (and thereby nested-paging) for
10887	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10888	* normally intercept #PFs, it might intercept them for debugging purposes.
10889	*
10890	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10891	* If the outer guest is intercepting #PFs we must intercept all #PFs.
10892	*/
10893	uint32_t u32XcptPFMask;
10894	uint32_t u32XcptPFMatch;
10895	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10896	{
10897	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10898	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10899	}
10900	else
10901	{
10902	u32XcptPFMask = 0;
10903	u32XcptPFMatch = 0;
10904	}
10905
10906	/*
10907	* Pause-Loop exiting.
10908	*/
10909	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10910	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10911
10912	/*
10913	* I/O Bitmap.
10914	*
10915	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we always
10916	* intercept all I/O port accesses.
10917	*/
10918	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10919
10920	/*
10921	* VMCS shadowing.
10922	*
10923	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10924	* enabled while executing the nested-guest.
10925	*/
10926	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10927
10928	/*
10929	* APIC-access page.
10930	*
10931	* The APIC-access page address has already been initialized while setting up the
10932	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it should
10933	* not be of any consequence to the host or to the guest for that matter, but we only
10934	* accept valid addresses verified by the VMLAUNCH/VMRESUME instruction emulation to keep
10935	* it simple.
10936	*/
10937
10938	/*
10939	* Virtual-APIC page and TPR threshold.
10940	*
10941	* We shall use the host-physical address of the virtual-APIC page in guest memory
10942	* directly. For this reason, we can access the virtual-APIC page of the nested-guest only
10943	* using PGM physical handlers as we must not assume a kernel virtual-address mapping
10944	* exists and requesting PGM for a mapping could be expensive/resource intensive (PGM
10945	* mapping cache).
10946	*/
10947	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10948	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10949	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10950	{
10951	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10952
10953	/*
10954	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10955	* we would fail to obtain a valid host-physical address for its guest-physical
10956	* address.
10957	*
10958	* We currently do not support this scenario. Maybe in the future if there is a
10959	* pressing need we can explore making this particular set of conditions work.
10960	* Right now we just cause a VM-entry failure.
10961	*
10962	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10963	* so should not really failure at the moment.
10964	*/
10965	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10966	}
10967	else
10968	{
10969	/*
10970	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10971	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10972	* be taken care of by EPT/shadow paging.
10973	*/
10974	if (pVM->hm.s.fAllow64BitGuests)
10975	{
10976	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10977	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10978	}
10979	}
10980
10981	/*
10982	* Validate basic assumptions.
10983	*/
10984	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10985	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10986	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10987	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10988
10989	/*
10990	* Commit it to the nested-guest VMCS.
10991	*/
10992	int rc = VINF_SUCCESS;
10993	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10994	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10995	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10996	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10997	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10998	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10999	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
11000	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
11001	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
11002	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
11003	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
11004	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
11005	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
11006	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
11007	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
11008	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
11009	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
11010	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
11011	{
11012	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
11013	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
11014	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
11015	}
11016	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
11017	{
11018	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
11019	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
11020	}
11021	AssertRCReturn(rc, rc);
11022
11023	/*
11024	* Update the nested-guest VMCS cache.
11025	*/
11026	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
11027	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
11028	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
11029	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
11030	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
11031	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
11032	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
11033	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
11034	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
11035
11036	/*
11037	* MSR bitmap.
11038	*
11039	* The MSR bitmap address has already been initialized while setting up the nested-guest
11040	* VMCS, here we need to merge the MSR bitmaps.
11041	*/
11042	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11043	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
11044
11045	return VINF_SUCCESS;
11046	}
11047	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11048
11049
11050	/**
11051	* Does the preparations before executing guest code in VT-x.
11052	*
11053	* This may cause longjmps to ring-3 and may even result in rescheduling to the
11054	* recompiler/IEM. We must be cautious what we do here regarding committing
11055	* guest-state information into the VMCS assuming we assuredly execute the
11056	* guest in VT-x mode.
11057	*
11058	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
11059	* the common-state (TRPM/forceflags), we must undo those changes so that the
11060	* recompiler/IEM can (and should) use them when it resumes guest execution.
11061	* Otherwise such operations must be done when we can no longer exit to ring-3.
11062	*
11063	* @returns Strict VBox status code (i.e. informational status codes too).
11064	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
11065	* have been disabled.
11066	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
11067	* pending events).
11068	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
11069	* double-fault into the guest.
11070	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
11071	* dispatched directly.
11072	* @retval VINF_* scheduling changes, we have to go back to ring-3.
11073	*
11074	* @param pVCpu The cross context virtual CPU structure.
11075	* @param pVmxTransient The VMX-transient structure.
11076	* @param fStepping Whether we are single-stepping the guest in the
11077	* hypervisor debugger. Makes us ignore some of the reasons
11078	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
11079	* if event dispatching took place.
11080	*/
11081	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
11082	{
11083	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11084
11085	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
11086	if (pVmxTransient->fIsNestedGuest)
11087	{
11088	RT_NOREF2(pVCpu, fStepping);
11089	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
11090	return VINF_EM_RESCHEDULE_REM;
11091	}
11092	#endif
11093
11094	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
11095	PGMRZDynMapFlushAutoSet(pVCpu);
11096	#endif
11097
11098	/*
11099	* Check and process force flag actions, some of which might require us to go back to ring-3.
11100	*/
11101	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
11102	if (rcStrict == VINF_SUCCESS)
11103	{ /* FFs don't get set all the time. */ }
11104	else
11105	return rcStrict;
11106
11107	/*
11108	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
11109	*/
11110	/** @todo Doing this from ring-3 after VM setup phase causes a
11111	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
11112	* idea why atm. */
11113	PVM pVM = pVCpu->CTX_SUFF(pVM);
11114	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
11115	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
11116	&& PDMHasApic(pVM))
11117	{
11118	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
11119	AssertRCReturn(rc, rc);
11120	}
11121
11122	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11123	/*
11124	* Merge guest VMCS controls with the nested-guest VMCS controls.
11125	*
11126	* Even if we have not executed the guest prior to this (e.g. when resuming from a
11127	* saved state), we should be okay with merging controls as we initialize the
11128	* guest VMCS controls as part of VM setup phase.
11129	*/
11130	if ( pVmxTransient->fIsNestedGuest
11131	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
11132	{
11133	int rc = hmR0VmxMergeVmcsNested(pVCpu);
11134	AssertRCReturn(rc, rc);
11135	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
11136	}
11137	#endif
11138
11139	/*
11140	* Evaluate events to be injected into the guest.
11141	*
11142	* Events in TRPM can be injected without inspecting the guest state.
11143	* If any new events (interrupts/NMI) are pending currently, we try to set up the
11144	* guest to cause a VM-exit the next time they are ready to receive the event.
11145	*
11146	* With nested-guests, evaluating pending events may cause VM-exits.
11147	*/
11148	if (TRPMHasTrap(pVCpu))
11149	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
11150
11151	uint32_t fIntrState;
11152	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
11153
11154	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11155	/*
11156	* While evaluating pending events if something failed (unlikely) or if we were
11157	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
11158	*/
11159	if (rcStrict != VINF_SUCCESS)
11160	return rcStrict;
11161	if ( pVmxTransient->fIsNestedGuest
11162	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11163	{
11164	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11165	return VINF_VMX_VMEXIT;
11166	}
11167	#else
11168	Assert(rcStrict == VINF_SUCCESS);
11169	#endif
11170
11171	/*
11172	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
11173	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
11174	* also result in triple-faulting the VM.
11175	*
11176	* With nested-guests, the above does not apply since unrestricted guest execution is a
11177	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
11178	*/
11179	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
11180	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11181	{ /* likely */ }
11182	else
11183	{
11184	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
11185	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11186	return rcStrict;
11187	}
11188
11189	/*
11190	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
11191	* import CR3 themselves. We will need to update them here, as even as late as the above
11192	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
11193	* the below force flags to be set.
11194	*/
11195	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
11196	{
11197	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
11198	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
11199	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
11200	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
11201	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11202	}
11203	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
11204	{
11205	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
11206	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11207	}
11208
11209	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11210	/* Paranoia. */
11211	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11212	#endif
11213
11214	/*
11215	* No longjmps to ring-3 from this point on!!!
11216	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
11217	* This also disables flushing of the R0-logger instance (if any).
11218	*/
11219	VMMRZCallRing3Disable(pVCpu);
11220
11221	/*
11222	* Export the guest state bits.
11223	*
11224	* We cannot perform longjmps while loading the guest state because we do not preserve the
11225	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
11226	* CPU migration.
11227	*
11228	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
11229	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
11230	*/
11231	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
11232	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11233	{ /* likely */ }
11234	else
11235	{
11236	VMMRZCallRing3Enable(pVCpu);
11237	return rcStrict;
11238	}
11239
11240	/*
11241	* We disable interrupts so that we don't miss any interrupts that would flag preemption
11242	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
11243	* preemption disabled for a while. Since this is purely to aid the
11244	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
11245	* disable interrupt on NT.
11246	*
11247	* We need to check for force-flags that could've possible been altered since we last
11248	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
11249	* see @bugref{6398}).
11250	*
11251	* We also check a couple of other force-flags as a last opportunity to get the EMT back
11252	* to ring-3 before executing guest code.
11253	*/
11254	pVmxTransient->fEFlags = ASMIntDisableFlags();
11255
11256	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
11257	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
11258	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
11259	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
11260	{
11261	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
11262	{
11263	pVCpu->hm.s.Event.fPending = false;
11264
11265	/*
11266	* We've injected any pending events. This is really the point of no return (to ring-3).
11267	*
11268	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
11269	* returns from this function, so don't enable them here.
11270	*/
11271	return VINF_SUCCESS;
11272	}
11273
11274	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
11275	rcStrict = VINF_EM_RAW_INTERRUPT;
11276	}
11277	else
11278	{
11279	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
11280	rcStrict = VINF_EM_RAW_TO_R3;
11281	}
11282
11283	ASMSetFlags(pVmxTransient->fEFlags);
11284	VMMRZCallRing3Enable(pVCpu);
11285
11286	return rcStrict;
11287	}
11288
11289
11290	/**
11291	* Final preparations before executing guest code using hardware-assisted VMX.
11292	*
11293	* We can no longer get preempted to a different host CPU and there are no returns
11294	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
11295	* failures), this function is not intended to fail sans unrecoverable hardware
11296	* errors.
11297	*
11298	* @param pVCpu The cross context virtual CPU structure.
11299	* @param pVmxTransient The VMX-transient structure.
11300	*
11301	* @remarks Called with preemption disabled.
11302	* @remarks No-long-jump zone!!!
11303	*/
11304	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11305	{
11306	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11307	Assert(VMMR0IsLogFlushDisabled(pVCpu));
11308	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
11309	Assert(!pVCpu->hm.s.Event.fPending);
11310
11311	/*
11312	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
11313	*/
11314	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11315	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
11316
11317	PVM pVM = pVCpu->CTX_SUFF(pVM);
11318	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11319	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
11320	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
11321
11322	if (!CPUMIsGuestFPUStateActive(pVCpu))
11323	{
11324	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11325	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
11326	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
11327	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11328	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
11329	}
11330
11331	/*
11332	* Re-export the host state bits as we may've been preempted (only happens when
11333	* thread-context hooks are used or when the VM start function changes) or if
11334	* the host CR0 is modified while loading the guest FPU state above.
11335	*
11336	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
11337	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
11338	* see @bugref{8432}.
11339	*
11340	* This may also happen when switching to/from a nested-guest VMCS without leaving
11341	* ring-0.
11342	*/
11343	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
11344	{
11345	hmR0VmxExportHostState(pVCpu);
11346	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
11347	}
11348	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
11349
11350	/*
11351	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
11352	*/
11353	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
11354	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
11355	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
11356
11357	/*
11358	* Store status of the shared guest/host debug state at the time of VM-entry.
11359	*/
11360	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
11361	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
11362	{
11363	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
11364	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
11365	}
11366	else
11367	#endif
11368	{
11369	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
11370	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
11371	}
11372
11373	/*
11374	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
11375	* more than one conditional check. The post-run side of our code shall determine
11376	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
11377	*/
11378	if (pVmcsInfo->pbVirtApic)
11379	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
11380
11381	/*
11382	* Update the host MSRs values in the VM-exit MSR-load area.
11383	*/
11384	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
11385	{
11386	if (pVmcsInfo->cExitMsrLoad > 0)
11387	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
11388	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
11389	}
11390
11391	/*
11392	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
11393	* VMX-preemption timer based on the next virtual sync clock deadline.
11394	*/
11395	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
11396	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
11397	{
11398	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
11399	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
11400	}
11401
11402	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
11403	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
11404	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
11405	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
11406	pVmcsInfo->idHostCpu = idCurrentCpu; /* Update the CPU for which we updated host-state in this VMCS. */
11407
11408	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
11409
11410	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
11411	as we're about to start executing the guest . */
11412
11413	/*
11414	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
11415	*
11416	* This is done this late as updating the TSC offsetting/preemption timer above
11417	* figures out if we can skip intercepting RDTSCP by calculating the number of
11418	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
11419	*/
11420	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
11421	&& !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11422	{
11423	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
11424
11425	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
11426	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
11427	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
11428	true /* fSetReadWrite /, true / fUpdateHostMsr */);
11429	AssertRC(rc);
11430	Assert(!pVmxTransient->fRemoveTscAuxMsr);
11431	pVmxTransient->fRemoveTscAuxMsr = true;
11432	}
11433
11434	#ifdef VBOX_STRICT
11435	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
11436	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11437	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
11438	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
11439	#endif
11440
11441	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
11442	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
11443	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
11444	* see @bugref{9180#c54}. */
11445	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
11446	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
11447	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
11448	#endif
11449	}
11450
11451
11452	/**
11453	* First C routine invoked after running guest code using hardware-assisted VMX.
11454	*
11455	* @param pVCpu The cross context virtual CPU structure.
11456	* @param pVmxTransient The VMX-transient structure.
11457	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
11458	*
11459	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
11460	*
11461	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
11462	* unconditionally when it is safe to do so.
11463	*/
11464	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
11465	{
11466	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
11467
11468	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
11469	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
11470	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
11471	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
11472	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
11473	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
11474
11475	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11476	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11477	{
11478	uint64_t uGstTsc;
11479	if (!pVmxTransient->fIsNestedGuest)
11480	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11481	else
11482	{
11483	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11484	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
11485	}
11486	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
11487	}
11488
11489	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11490	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
11491	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11492
11493	#if HC_ARCH_BITS == 64
11494	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11495	#endif
11496	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
11497	/* The 64-on-32 switcher maintains VMCS-launch state on its own
11498	and we need to leave it alone here. */
11499	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
11500	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11501	#else
11502	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11503	#endif
11504	#ifdef VBOX_STRICT
11505	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11506	#endif
11507	Assert(!ASMIntAreEnabled());
11508	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11509	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11510
11511	/*
11512	* Save the basic VM-exit reason and check if the VM-entry failed.
11513	* See Intel spec. 24.9.1 "Basic VM-exit Information".
11514	*/
11515	uint32_t uExitReason;
11516	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11517	AssertRC(rc);
11518	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11519	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11520
11521	/*
11522	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11523	* bitmap permissions, if it was added before VM-entry.
11524	*/
11525	if (pVmxTransient->fRemoveTscAuxMsr)
11526	{
11527	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11528	pVmxTransient->fRemoveTscAuxMsr = false;
11529	}
11530
11531	/*
11532	* Check if VMLAUNCH/VMRESUME succeeded.
11533	* If this failed, we cause a guru meditation and cease further execution.
11534	*
11535	* However, if we are executing a nested-guest we might fail if we use the
11536	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11537	*/
11538	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11539	{
11540	/*
11541	* Update the VM-exit history array here even if the VM-entry failed due to:
11542	* - Invalid guest state.
11543	* - MSR loading.
11544	* - Machine-check event.
11545	*
11546	* In any of the above cases we will still have a "valid" VM-exit reason
11547	* despite @a fVMEntryFailed being false.
11548	*
11549	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11550	*
11551	* Note! We don't have CS or RIP at this point. Will probably address that later
11552	* by amending the history entry added here.
11553	*/
11554	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11555	UINT64_MAX, uHostTsc);
11556
11557	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11558	{
11559	VMMRZCallRing3Enable(pVCpu);
11560
11561	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11562	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11563
11564	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11565	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11566	AssertRC(rc);
11567	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11568	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11569	AssertRC(rc);
11570	#else
11571	/*
11572	* Import the guest-interruptibility state always as we need it while evaluating
11573	* injecting events on re-entry.
11574	*
11575	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11576	* checking for real-mode while exporting the state because all bits that cause
11577	* mode changes wrt CR0 are intercepted.
11578	*/
11579	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11580	AssertRC(rc);
11581	#endif
11582
11583	/*
11584	* Sync the TPR shadow with our APIC state.
11585	*/
11586	if ( !pVmxTransient->fIsNestedGuest
11587	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
11588	{
11589	Assert(pVmcsInfo->pbVirtApic);
11590	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11591	{
11592	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11593	AssertRC(rc);
11594	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11595	}
11596	}
11597
11598	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11599	return;
11600	}
11601	}
11602	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11603	else if (pVmxTransient->fIsNestedGuest)
11604	{
11605	# if 0
11606	/*
11607	* Copy the VM-instruction error field to the guest VMCS.
11608	*/
11609	/** @todo NSTVMX: Verify we're using the fast path. */
11610	uint32_t u32RoVmInstrError;
11611	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
11612	AssertRCReturn(rc, rc);
11613	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
11614	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
11615	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
11616	# else
11617	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11618	# endif
11619	}
11620	#endif
11621	else
11622	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11623
11624	VMMRZCallRing3Enable(pVCpu);
11625	}
11626
11627
11628	/**
11629	* Runs the guest code using hardware-assisted VMX the normal way.
11630	*
11631	* @returns VBox status code.
11632	* @param pVCpu The cross context virtual CPU structure.
11633	* @param pcLoops Pointer to the number of executed loops.
11634	*/
11635	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11636	{
11637	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11638	Assert(pcLoops);
11639	Assert(*pcLoops <= cMaxResumeLoops);
11640	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11641
11642	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11643	/*
11644	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11645	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11646	* guest VMCS while entering the VMX ring-0 session.
11647	*/
11648	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11649	{
11650	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11651	if (RT_SUCCESS(rc))
11652	{ /* likely */ }
11653	else
11654	{
11655	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11656	return rc;
11657	}
11658	}
11659	#endif
11660
11661	VMXTRANSIENT VmxTransient;
11662	RT_ZERO(VmxTransient);
11663	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11664
11665	/* Paranoia. */
11666	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11667
11668	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11669	for (;;)
11670	{
11671	Assert(!HMR0SuspendPending());
11672	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11673	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11674
11675	/*
11676	* Preparatory work for running nested-guest code, this may force us to
11677	* return to ring-3.
11678	*
11679	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11680	*/
11681	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11682	if (rcStrict != VINF_SUCCESS)
11683	break;
11684
11685	/* Interrupts are disabled at this point! */
11686	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11687	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11688	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11689	/* Interrupts are re-enabled at this point! */
11690
11691	/*
11692	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11693	*/
11694	if (RT_SUCCESS(rcRun))
11695	{ /* very likely */ }
11696	else
11697	{
11698	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11699	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11700	return rcRun;
11701	}
11702
11703	/*
11704	* Profile the VM-exit.
11705	*/
11706	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11707	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11708	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11709	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11710	HMVMX_START_EXIT_DISPATCH_PROF();
11711
11712	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11713
11714	/*
11715	* Handle the VM-exit.
11716	*/
11717	#ifdef HMVMX_USE_FUNCTION_TABLE
11718	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11719	#else
11720	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11721	#endif
11722	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11723	if (rcStrict == VINF_SUCCESS)
11724	{
11725	if (++(*pcLoops) <= cMaxResumeLoops)
11726	continue;
11727	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11728	rcStrict = VINF_EM_RAW_INTERRUPT;
11729	}
11730	break;
11731	}
11732
11733	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11734	return rcStrict;
11735	}
11736
11737
11738	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11739	/**
11740	* Runs the nested-guest code using hardware-assisted VMX.
11741	*
11742	* @returns VBox status code.
11743	* @param pVCpu The cross context virtual CPU structure.
11744	* @param pcLoops Pointer to the number of executed loops.
11745	*
11746	* @sa hmR0VmxRunGuestCodeNormal.
11747	*/
11748	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11749	{
11750	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11751	Assert(pcLoops);
11752	Assert(*pcLoops <= cMaxResumeLoops);
11753	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11754
11755	/*
11756	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11757	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11758	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11759	*/
11760	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11761	{
11762	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11763	if (RT_SUCCESS(rc))
11764	{ /* likely */ }
11765	else
11766	{
11767	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11768	return rc;
11769	}
11770	}
11771
11772	VMXTRANSIENT VmxTransient;
11773	RT_ZERO(VmxTransient);
11774	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11775	VmxTransient.fIsNestedGuest = true;
11776
11777	/* Paranoia. */
11778	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11779
11780	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11781	for (;;)
11782	{
11783	Assert(!HMR0SuspendPending());
11784	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11785	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11786
11787	/*
11788	* Preparatory work for running guest code, this may force us to
11789	* return to ring-3.
11790	*
11791	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11792	*/
11793	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11794	if (rcStrict != VINF_SUCCESS)
11795	break;
11796
11797	/* Interrupts are disabled at this point! */
11798	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11799	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11800	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11801	/* Interrupts are re-enabled at this point! */
11802
11803	/*
11804	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11805	*/
11806	if (RT_SUCCESS(rcRun))
11807	{ /* very likely */ }
11808	else
11809	{
11810	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11811	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11812	return rcRun;
11813	}
11814
11815	/*
11816	* Profile the VM-exit.
11817	*/
11818	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11819	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11820	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11821	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11822	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11823	HMVMX_START_EXIT_DISPATCH_PROF();
11824
11825	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11826
11827	/*
11828	* Handle the VM-exit.
11829	*/
11830	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11831	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11832	if (rcStrict == VINF_SUCCESS)
11833	{
11834	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11835	{
11836	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11837	rcStrict = VINF_VMX_VMEXIT;
11838	}
11839	else
11840	{
11841	if (++(*pcLoops) <= cMaxResumeLoops)
11842	continue;
11843	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11844	rcStrict = VINF_EM_RAW_INTERRUPT;
11845	}
11846	}
11847	else
11848	Assert(rcStrict != VINF_VMX_VMEXIT);
11849	break;
11850	}
11851
11852	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11853	return rcStrict;
11854	}
11855	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11856
11857
11858	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11859	* probes.
11860	*
11861	* The following few functions and associated structure contains the bloat
11862	* necessary for providing detailed debug events and dtrace probes as well as
11863	* reliable host side single stepping. This works on the principle of
11864	* "subclassing" the normal execution loop and workers. We replace the loop
11865	* method completely and override selected helpers to add necessary adjustments
11866	* to their core operation.
11867	*
11868	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11869	* any performance for debug and analysis features.
11870	*
11871	* @{
11872	*/
11873
11874	/**
11875	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11876	* the debug run loop.
11877	*/
11878	typedef struct VMXRUNDBGSTATE
11879	{
11880	/** The RIP we started executing at. This is for detecting that we stepped. */
11881	uint64_t uRipStart;
11882	/** The CS we started executing with. */
11883	uint16_t uCsStart;
11884
11885	/** Whether we've actually modified the 1st execution control field. */
11886	bool fModifiedProcCtls : 1;
11887	/** Whether we've actually modified the 2nd execution control field. */
11888	bool fModifiedProcCtls2 : 1;
11889	/** Whether we've actually modified the exception bitmap. */
11890	bool fModifiedXcptBitmap : 1;
11891
11892	/** We desire the modified the CR0 mask to be cleared. */
11893	bool fClearCr0Mask : 1;
11894	/** We desire the modified the CR4 mask to be cleared. */
11895	bool fClearCr4Mask : 1;
11896	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11897	uint32_t fCpe1Extra;
11898	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11899	uint32_t fCpe1Unwanted;
11900	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11901	uint32_t fCpe2Extra;
11902	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11903	uint32_t bmXcptExtra;
11904	/** The sequence number of the Dtrace provider settings the state was
11905	* configured against. */
11906	uint32_t uDtraceSettingsSeqNo;
11907	/** VM-exits to check (one bit per VM-exit). */
11908	uint32_t bmExitsToCheck[3];
11909
11910	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11911	uint32_t fProcCtlsInitial;
11912	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11913	uint32_t fProcCtls2Initial;
11914	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11915	uint32_t bmXcptInitial;
11916	} VMXRUNDBGSTATE;
11917	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11918	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11919
11920
11921	/**
11922	* Initializes the VMXRUNDBGSTATE structure.
11923	*
11924	* @param pVCpu The cross context virtual CPU structure of the
11925	* calling EMT.
11926	* @param pVmxTransient The VMX-transient structure.
11927	* @param pDbgState The debug state to initialize.
11928	*/
11929	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11930	{
11931	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11932	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11933
11934	pDbgState->fModifiedProcCtls = false;
11935	pDbgState->fModifiedProcCtls2 = false;
11936	pDbgState->fModifiedXcptBitmap = false;
11937	pDbgState->fClearCr0Mask = false;
11938	pDbgState->fClearCr4Mask = false;
11939	pDbgState->fCpe1Extra = 0;
11940	pDbgState->fCpe1Unwanted = 0;
11941	pDbgState->fCpe2Extra = 0;
11942	pDbgState->bmXcptExtra = 0;
11943	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11944	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11945	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11946	}
11947
11948
11949	/**
11950	* Updates the VMSC fields with changes requested by @a pDbgState.
11951	*
11952	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11953	* immediately before executing guest code, i.e. when interrupts are disabled.
11954	* We don't check status codes here as we cannot easily assert or return in the
11955	* latter case.
11956	*
11957	* @param pVCpu The cross context virtual CPU structure.
11958	* @param pVmxTransient The VMX-transient structure.
11959	* @param pDbgState The debug state.
11960	*/
11961	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11962	{
11963	/*
11964	* Ensure desired flags in VMCS control fields are set.
11965	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11966	*
11967	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11968	* there should be no stale data in pCtx at this point.
11969	*/
11970	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11971	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11972	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11973	{
11974	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11975	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11976	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11977	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11978	pDbgState->fModifiedProcCtls = true;
11979	}
11980
11981	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11982	{
11983	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11984	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11985	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11986	pDbgState->fModifiedProcCtls2 = true;
11987	}
11988
11989	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11990	{
11991	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11992	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11993	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11994	pDbgState->fModifiedXcptBitmap = true;
11995	}
11996
11997	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11998	{
11999	pVmcsInfo->u64Cr0Mask = 0;
12000	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
12001	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
12002	}
12003
12004	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
12005	{
12006	pVmcsInfo->u64Cr4Mask = 0;
12007	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
12008	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
12009	}
12010
12011	NOREF(pVCpu);
12012	}
12013
12014
12015	/**
12016	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
12017	* re-entry next time around.
12018	*
12019	* @returns Strict VBox status code (i.e. informational status codes too).
12020	* @param pVCpu The cross context virtual CPU structure.
12021	* @param pVmxTransient The VMX-transient structure.
12022	* @param pDbgState The debug state.
12023	* @param rcStrict The return code from executing the guest using single
12024	* stepping.
12025	*/
12026	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
12027	VBOXSTRICTRC rcStrict)
12028	{
12029	/*
12030	* Restore VM-exit control settings as we may not reenter this function the
12031	* next time around.
12032	*/
12033	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12034
12035	/* We reload the initial value, trigger what we can of recalculations the
12036	next time around. From the looks of things, that's all that's required atm. */
12037	if (pDbgState->fModifiedProcCtls)
12038	{
12039	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
12040	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
12041	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
12042	AssertRCReturn(rc2, rc2);
12043	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
12044	}
12045
12046	/* We're currently the only ones messing with this one, so just restore the
12047	cached value and reload the field. */
12048	if ( pDbgState->fModifiedProcCtls2
12049	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
12050	{
12051	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
12052	AssertRCReturn(rc2, rc2);
12053	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
12054	}
12055
12056	/* If we've modified the exception bitmap, we restore it and trigger
12057	reloading and partial recalculation the next time around. */
12058	if (pDbgState->fModifiedXcptBitmap)
12059	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
12060
12061	return rcStrict;
12062	}
12063
12064
12065	/**
12066	* Configures VM-exit controls for current DBGF and DTrace settings.
12067	*
12068	* This updates @a pDbgState and the VMCS execution control fields to reflect
12069	* the necessary VM-exits demanded by DBGF and DTrace.
12070	*
12071	* @param pVCpu The cross context virtual CPU structure.
12072	* @param pVmxTransient The VMX-transient structure. May update
12073	* fUpdatedTscOffsettingAndPreemptTimer.
12074	* @param pDbgState The debug state.
12075	*/
12076	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12077	{
12078	/*
12079	* Take down the dtrace serial number so we can spot changes.
12080	*/
12081	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
12082	ASMCompilerBarrier();
12083
12084	/*
12085	* We'll rebuild most of the middle block of data members (holding the
12086	* current settings) as we go along here, so start by clearing it all.
12087	*/
12088	pDbgState->bmXcptExtra = 0;
12089	pDbgState->fCpe1Extra = 0;
12090	pDbgState->fCpe1Unwanted = 0;
12091	pDbgState->fCpe2Extra = 0;
12092	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
12093	pDbgState->bmExitsToCheck[i] = 0;
12094
12095	/*
12096	* Software interrupts (INT XXh) - no idea how to trigger these...
12097	*/
12098	PVM pVM = pVCpu->CTX_SUFF(pVM);
12099	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
12100	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
12101	{
12102	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12103	}
12104
12105	/*
12106	* INT3 breakpoints - triggered by #BP exceptions.
12107	*/
12108	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
12109	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12110
12111	/*
12112	* Exception bitmap and XCPT events+probes.
12113	*/
12114	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
12115	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
12116	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
12117
12118	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
12119	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
12120	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12121	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
12122	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
12123	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
12124	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
12125	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
12126	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
12127	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
12128	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
12129	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
12130	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
12131	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
12132	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
12133	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
12134	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
12135	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
12136
12137	if (pDbgState->bmXcptExtra)
12138	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12139
12140	/*
12141	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
12142	*
12143	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
12144	* So, when adding/changing/removing please don't forget to update it.
12145	*
12146	* Some of the macros are picking up local variables to save horizontal space,
12147	* (being able to see it in a table is the lesser evil here).
12148	*/
12149	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
12150	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
12151	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
12152	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
12153	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12154	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12155	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12156	} else do { } while (0)
12157	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
12158	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12159	{ \
12160	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
12161	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12162	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12163	} else do { } while (0)
12164	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
12165	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12166	{ \
12167	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
12168	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12169	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12170	} else do { } while (0)
12171	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
12172	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12173	{ \
12174	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
12175	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12176	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12177	} else do { } while (0)
12178
12179	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
12180	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
12181	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
12182	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
12183	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
12184
12185	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
12186	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
12187	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
12188	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
12189	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
12190	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
12191	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
12192	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
12193	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
12194	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
12195	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
12196	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
12197	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
12198	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
12199	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
12200	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
12201	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
12202	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
12203	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
12204	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
12205	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
12206	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
12207	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
12208	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
12209	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
12210	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
12211	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
12212	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
12213	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
12214	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
12215	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
12216	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
12217	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
12218	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
12219	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
12220	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
12221
12222	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
12223	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12224	{
12225	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
12226	\| CPUMCTX_EXTRN_APIC_TPR);
12227	AssertRC(rc);
12228
12229	#if 0 /** @todo fix me */
12230	pDbgState->fClearCr0Mask = true;
12231	pDbgState->fClearCr4Mask = true;
12232	#endif
12233	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
12234	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
12235	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12236	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
12237	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
12238	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
12239	require clearing here and in the loop if we start using it. */
12240	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
12241	}
12242	else
12243	{
12244	if (pDbgState->fClearCr0Mask)
12245	{
12246	pDbgState->fClearCr0Mask = false;
12247	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12248	}
12249	if (pDbgState->fClearCr4Mask)
12250	{
12251	pDbgState->fClearCr4Mask = false;
12252	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12253	}
12254	}
12255	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
12256	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
12257
12258	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
12259	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
12260	{
12261	/** @todo later, need to fix handler as it assumes this won't usually happen. */
12262	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
12263	}
12264	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
12265	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
12266
12267	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
12268	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
12269	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
12270	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
12271	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
12272	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
12273	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
12274	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
12275	#if 0 /** @todo too slow, fix handler. */
12276	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
12277	#endif
12278	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
12279
12280	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
12281	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
12282	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
12283	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
12284	{
12285	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12286	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
12287	}
12288	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12289	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12290	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12291	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12292
12293	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
12294	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
12295	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
12296	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
12297	{
12298	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12299	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
12300	}
12301	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
12302	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
12303	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
12304	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
12305
12306	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
12307	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
12308	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
12309	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
12310	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
12311	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
12312	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
12313	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
12314	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
12315	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
12316	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
12317	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
12318	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
12319	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
12320	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
12321	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
12322	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
12323	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
12324	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
12325	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
12326	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
12327	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
12328
12329	#undef IS_EITHER_ENABLED
12330	#undef SET_ONLY_XBM_IF_EITHER_EN
12331	#undef SET_CPE1_XBM_IF_EITHER_EN
12332	#undef SET_CPEU_XBM_IF_EITHER_EN
12333	#undef SET_CPE2_XBM_IF_EITHER_EN
12334
12335	/*
12336	* Sanitize the control stuff.
12337	*/
12338	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
12339	if (pDbgState->fCpe2Extra)
12340	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
12341	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
12342	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
12343	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
12344	{
12345	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
12346	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12347	}
12348
12349	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
12350	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
12351	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
12352	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
12353	}
12354
12355
12356	/**
12357	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
12358	* appropriate.
12359	*
12360	* The caller has checked the VM-exit against the
12361	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
12362	* already, so we don't have to do that either.
12363	*
12364	* @returns Strict VBox status code (i.e. informational status codes too).
12365	* @param pVCpu The cross context virtual CPU structure.
12366	* @param pVmxTransient The VMX-transient structure.
12367	* @param uExitReason The VM-exit reason.
12368	*
12369	* @remarks The name of this function is displayed by dtrace, so keep it short
12370	* and to the point. No longer than 33 chars long, please.
12371	*/
12372	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
12373	{
12374	/*
12375	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
12376	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
12377	*
12378	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
12379	* does. Must add/change/remove both places. Same ordering, please.
12380	*
12381	* Added/removed events must also be reflected in the next section
12382	* where we dispatch dtrace events.
12383	*/
12384	bool fDtrace1 = false;
12385	bool fDtrace2 = false;
12386	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
12387	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
12388	uint32_t uEventArg = 0;
12389	#define SET_EXIT(a_EventSubName) \
12390	do { \
12391	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12392	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12393	} while (0)
12394	#define SET_BOTH(a_EventSubName) \
12395	do { \
12396	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
12397	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12398	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
12399	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12400	} while (0)
12401	switch (uExitReason)
12402	{
12403	case VMX_EXIT_MTF:
12404	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12405
12406	case VMX_EXIT_XCPT_OR_NMI:
12407	{
12408	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12409	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
12410	{
12411	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12412	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
12413	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
12414	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
12415	{
12416	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
12417	{
12418	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12419	uEventArg = pVmxTransient->uExitIntErrorCode;
12420	}
12421	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
12422	switch (enmEvent1)
12423	{
12424	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
12425	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
12426	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
12427	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
12428	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
12429	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
12430	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
12431	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
12432	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
12433	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
12434	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
12435	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
12436	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
12437	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
12438	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
12439	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
12440	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
12441	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
12442	default: break;
12443	}
12444	}
12445	else
12446	AssertFailed();
12447	break;
12448
12449	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
12450	uEventArg = idxVector;
12451	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
12452	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
12453	break;
12454	}
12455	break;
12456	}
12457
12458	case VMX_EXIT_TRIPLE_FAULT:
12459	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
12460	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
12461	break;
12462	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
12463	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
12464	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
12465	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
12466	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
12467
12468	/* Instruction specific VM-exits: */
12469	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
12470	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
12471	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
12472	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
12473	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
12474	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
12475	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
12476	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
12477	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
12478	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
12479	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
12480	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
12481	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
12482	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
12483	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
12484	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
12485	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12486	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12487	case VMX_EXIT_MOV_CRX:
12488	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12489	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12490	SET_BOTH(CRX_READ);
12491	else
12492	SET_BOTH(CRX_WRITE);
12493	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12494	break;
12495	case VMX_EXIT_MOV_DRX:
12496	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12497	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12498	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12499	SET_BOTH(DRX_READ);
12500	else
12501	SET_BOTH(DRX_WRITE);
12502	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12503	break;
12504	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12505	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12506	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12507	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12508	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12509	case VMX_EXIT_GDTR_IDTR_ACCESS:
12510	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12511	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12512	{
12513	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12514	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12515	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12516	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12517	}
12518	break;
12519
12520	case VMX_EXIT_LDTR_TR_ACCESS:
12521	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12522	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12523	{
12524	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12525	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12526	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12527	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12528	}
12529	break;
12530
12531	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12532	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12533	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12534	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12535	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12536	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12537	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12538	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12539	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12540	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12541	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12542
12543	/* Events that aren't relevant at this point. */
12544	case VMX_EXIT_EXT_INT:
12545	case VMX_EXIT_INT_WINDOW:
12546	case VMX_EXIT_NMI_WINDOW:
12547	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12548	case VMX_EXIT_PREEMPT_TIMER:
12549	case VMX_EXIT_IO_INSTR:
12550	break;
12551
12552	/* Errors and unexpected events. */
12553	case VMX_EXIT_INIT_SIGNAL:
12554	case VMX_EXIT_SIPI:
12555	case VMX_EXIT_IO_SMI:
12556	case VMX_EXIT_SMI:
12557	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12558	case VMX_EXIT_ERR_MSR_LOAD:
12559	case VMX_EXIT_ERR_MACHINE_CHECK:
12560	case VMX_EXIT_PML_FULL:
12561	case VMX_EXIT_VIRTUALIZED_EOI:
12562	break;
12563
12564	default:
12565	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12566	break;
12567	}
12568	#undef SET_BOTH
12569	#undef SET_EXIT
12570
12571	/*
12572	* Dtrace tracepoints go first. We do them here at once so we don't
12573	* have to copy the guest state saving and stuff a few dozen times.
12574	* Down side is that we've got to repeat the switch, though this time
12575	* we use enmEvent since the probes are a subset of what DBGF does.
12576	*/
12577	if (fDtrace1 \|\| fDtrace2)
12578	{
12579	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12580	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12581	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12582	switch (enmEvent1)
12583	{
12584	/** @todo consider which extra parameters would be helpful for each probe. */
12585	case DBGFEVENT_END: break;
12586	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12587	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12588	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12589	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12590	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12591	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12592	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12593	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12594	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12595	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12596	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12597	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12598	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12599	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12600	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12601	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12602	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12603	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12604	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12605	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12606	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12607	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12608	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12609	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12610	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12611	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12612	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12613	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12614	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12615	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12616	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12617	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12618	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12619	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12620	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12621	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12622	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12623	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12624	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12625	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12626	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12627	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12628	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12629	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12630	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12631	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12632	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12633	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12634	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12635	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12636	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12637	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12638	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12639	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12640	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12641	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12642	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12643	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12644	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12645	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12646	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12647	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12648	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12649	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12650	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12651	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12652	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12653	}
12654	switch (enmEvent2)
12655	{
12656	/** @todo consider which extra parameters would be helpful for each probe. */
12657	case DBGFEVENT_END: break;
12658	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12659	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12660	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12661	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12662	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12663	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12664	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12665	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12666	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12667	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12668	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12669	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12670	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12671	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12672	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12673	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12674	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12675	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12676	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12677	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12678	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12679	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12680	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12681	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12682	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12683	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12684	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12685	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12686	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12687	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12688	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12689	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12690	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12691	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12692	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12693	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12694	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12695	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12696	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12697	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12698	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12699	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12700	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12701	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12702	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12703	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12704	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12705	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12706	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12707	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12708	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12709	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12710	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12711	}
12712	}
12713
12714	/*
12715	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12716	* the DBGF call will do a full check).
12717	*
12718	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12719	* Note! If we have to events, we prioritize the first, i.e. the instruction
12720	* one, in order to avoid event nesting.
12721	*/
12722	PVM pVM = pVCpu->CTX_SUFF(pVM);
12723	if ( enmEvent1 != DBGFEVENT_END
12724	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12725	{
12726	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12727	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12728	if (rcStrict != VINF_SUCCESS)
12729	return rcStrict;
12730	}
12731	else if ( enmEvent2 != DBGFEVENT_END
12732	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12733	{
12734	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12735	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12736	if (rcStrict != VINF_SUCCESS)
12737	return rcStrict;
12738	}
12739
12740	return VINF_SUCCESS;
12741	}
12742
12743
12744	/**
12745	* Single-stepping VM-exit filtering.
12746	*
12747	* This is preprocessing the VM-exits and deciding whether we've gotten far
12748	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12749	* handling is performed.
12750	*
12751	* @returns Strict VBox status code (i.e. informational status codes too).
12752	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12753	* @param pVmxTransient The VMX-transient structure.
12754	* @param pDbgState The debug state.
12755	*/
12756	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12757	{
12758	/*
12759	* Expensive (saves context) generic dtrace VM-exit probe.
12760	*/
12761	uint32_t const uExitReason = pVmxTransient->uExitReason;
12762	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12763	{ /* more likely */ }
12764	else
12765	{
12766	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12767	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12768	AssertRC(rc);
12769	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12770	}
12771
12772	/*
12773	* Check for host NMI, just to get that out of the way.
12774	*/
12775	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12776	{ /* normally likely */ }
12777	else
12778	{
12779	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12780	AssertRCReturn(rc2, rc2);
12781	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12782	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12783	return hmR0VmxExitHostNmi(pVCpu);
12784	}
12785
12786	/*
12787	* Check for single stepping event if we're stepping.
12788	*/
12789	if (pVCpu->hm.s.fSingleInstruction)
12790	{
12791	switch (uExitReason)
12792	{
12793	case VMX_EXIT_MTF:
12794	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12795
12796	/* Various events: */
12797	case VMX_EXIT_XCPT_OR_NMI:
12798	case VMX_EXIT_EXT_INT:
12799	case VMX_EXIT_TRIPLE_FAULT:
12800	case VMX_EXIT_INT_WINDOW:
12801	case VMX_EXIT_NMI_WINDOW:
12802	case VMX_EXIT_TASK_SWITCH:
12803	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12804	case VMX_EXIT_APIC_ACCESS:
12805	case VMX_EXIT_EPT_VIOLATION:
12806	case VMX_EXIT_EPT_MISCONFIG:
12807	case VMX_EXIT_PREEMPT_TIMER:
12808
12809	/* Instruction specific VM-exits: */
12810	case VMX_EXIT_CPUID:
12811	case VMX_EXIT_GETSEC:
12812	case VMX_EXIT_HLT:
12813	case VMX_EXIT_INVD:
12814	case VMX_EXIT_INVLPG:
12815	case VMX_EXIT_RDPMC:
12816	case VMX_EXIT_RDTSC:
12817	case VMX_EXIT_RSM:
12818	case VMX_EXIT_VMCALL:
12819	case VMX_EXIT_VMCLEAR:
12820	case VMX_EXIT_VMLAUNCH:
12821	case VMX_EXIT_VMPTRLD:
12822	case VMX_EXIT_VMPTRST:
12823	case VMX_EXIT_VMREAD:
12824	case VMX_EXIT_VMRESUME:
12825	case VMX_EXIT_VMWRITE:
12826	case VMX_EXIT_VMXOFF:
12827	case VMX_EXIT_VMXON:
12828	case VMX_EXIT_MOV_CRX:
12829	case VMX_EXIT_MOV_DRX:
12830	case VMX_EXIT_IO_INSTR:
12831	case VMX_EXIT_RDMSR:
12832	case VMX_EXIT_WRMSR:
12833	case VMX_EXIT_MWAIT:
12834	case VMX_EXIT_MONITOR:
12835	case VMX_EXIT_PAUSE:
12836	case VMX_EXIT_GDTR_IDTR_ACCESS:
12837	case VMX_EXIT_LDTR_TR_ACCESS:
12838	case VMX_EXIT_INVEPT:
12839	case VMX_EXIT_RDTSCP:
12840	case VMX_EXIT_INVVPID:
12841	case VMX_EXIT_WBINVD:
12842	case VMX_EXIT_XSETBV:
12843	case VMX_EXIT_RDRAND:
12844	case VMX_EXIT_INVPCID:
12845	case VMX_EXIT_VMFUNC:
12846	case VMX_EXIT_RDSEED:
12847	case VMX_EXIT_XSAVES:
12848	case VMX_EXIT_XRSTORS:
12849	{
12850	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12851	AssertRCReturn(rc, rc);
12852	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12853	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12854	return VINF_EM_DBG_STEPPED;
12855	break;
12856	}
12857
12858	/* Errors and unexpected events: */
12859	case VMX_EXIT_INIT_SIGNAL:
12860	case VMX_EXIT_SIPI:
12861	case VMX_EXIT_IO_SMI:
12862	case VMX_EXIT_SMI:
12863	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12864	case VMX_EXIT_ERR_MSR_LOAD:
12865	case VMX_EXIT_ERR_MACHINE_CHECK:
12866	case VMX_EXIT_PML_FULL:
12867	case VMX_EXIT_VIRTUALIZED_EOI:
12868	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12869	break;
12870
12871	default:
12872	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12873	break;
12874	}
12875	}
12876
12877	/*
12878	* Check for debugger event breakpoints and dtrace probes.
12879	*/
12880	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12881	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12882	{
12883	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12884	if (rcStrict != VINF_SUCCESS)
12885	return rcStrict;
12886	}
12887
12888	/*
12889	* Normal processing.
12890	*/
12891	#ifdef HMVMX_USE_FUNCTION_TABLE
12892	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12893	#else
12894	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12895	#endif
12896	}
12897
12898
12899	/**
12900	* Single steps guest code using hardware-assisted VMX.
12901	*
12902	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12903	* but single-stepping through the hypervisor debugger.
12904	*
12905	* @returns Strict VBox status code (i.e. informational status codes too).
12906	* @param pVCpu The cross context virtual CPU structure.
12907	* @param pcLoops Pointer to the number of executed loops.
12908	*
12909	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12910	*/
12911	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12912	{
12913	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12914	Assert(pcLoops);
12915	Assert(*pcLoops <= cMaxResumeLoops);
12916
12917	VMXTRANSIENT VmxTransient;
12918	RT_ZERO(VmxTransient);
12919	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12920
12921	/* Set HMCPU indicators. */
12922	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12923	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12924	pVCpu->hm.s.fDebugWantRdTscExit = false;
12925	pVCpu->hm.s.fUsingDebugLoop = true;
12926
12927	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12928	VMXRUNDBGSTATE DbgState;
12929	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12930	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12931
12932	/*
12933	* The loop.
12934	*/
12935	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12936	for (;;)
12937	{
12938	Assert(!HMR0SuspendPending());
12939	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12940	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12941	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12942
12943	/* Set up VM-execution controls the next two can respond to. */
12944	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12945
12946	/*
12947	* Preparatory work for running guest code, this may force us to
12948	* return to ring-3.
12949	*
12950	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12951	*/
12952	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12953	if (rcStrict != VINF_SUCCESS)
12954	break;
12955
12956	/* Interrupts are disabled at this point! */
12957	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12958
12959	/* Override any obnoxious code in the above two calls. */
12960	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12961
12962	/*
12963	* Finally execute the guest.
12964	*/
12965	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12966
12967	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12968	/* Interrupts are re-enabled at this point! */
12969
12970	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12971	if (RT_SUCCESS(rcRun))
12972	{ /* very likely */ }
12973	else
12974	{
12975	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12976	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12977	return rcRun;
12978	}
12979
12980	/* Profile the VM-exit. */
12981	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12983	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12984	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12985	HMVMX_START_EXIT_DISPATCH_PROF();
12986
12987	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12988
12989	/*
12990	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12991	*/
12992	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12993	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12994	if (rcStrict != VINF_SUCCESS)
12995	break;
12996	if (++(*pcLoops) > cMaxResumeLoops)
12997	{
12998	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12999	rcStrict = VINF_EM_RAW_INTERRUPT;
13000	break;
13001	}
13002
13003	/*
13004	* Stepping: Did the RIP change, if so, consider it a single step.
13005	* Otherwise, make sure one of the TFs gets set.
13006	*/
13007	if (fStepping)
13008	{
13009	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13010	AssertRC(rc);
13011	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
13012	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
13013	{
13014	rcStrict = VINF_EM_DBG_STEPPED;
13015	break;
13016	}
13017	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
13018	}
13019
13020	/*
13021	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
13022	*/
13023	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
13024	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
13025	}
13026
13027	/*
13028	* Clear the X86_EFL_TF if necessary.
13029	*/
13030	if (pVCpu->hm.s.fClearTrapFlag)
13031	{
13032	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
13033	AssertRC(rc);
13034	pVCpu->hm.s.fClearTrapFlag = false;
13035	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
13036	}
13037	/** @todo there seems to be issues with the resume flag when the monitor trap
13038	* flag is pending without being used. Seen early in bios init when
13039	* accessing APIC page in protected mode. */
13040
13041	/*
13042	* Restore VM-exit control settings as we may not re-enter this function the
13043	* next time around.
13044	*/
13045	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
13046
13047	/* Restore HMCPU indicators. */
13048	pVCpu->hm.s.fUsingDebugLoop = false;
13049	pVCpu->hm.s.fDebugWantRdTscExit = false;
13050	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
13051
13052	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
13053	return rcStrict;
13054	}
13055
13056
13057	/** @} */
13058
13059
13060	/**
13061	* Checks if any expensive dtrace probes are enabled and we should go to the
13062	* debug loop.
13063	*
13064	* @returns true if we should use debug loop, false if not.
13065	*/
13066	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
13067	{
13068	/* It's probably faster to OR the raw 32-bit counter variables together.
13069	Since the variables are in an array and the probes are next to one
13070	another (more or less), we have good locality. So, better read
13071	eight-nine cache lines ever time and only have one conditional, than
13072	128+ conditionals, right? */
13073	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
13074	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
13075	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
13076	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
13077	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
13078	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
13079	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
13080	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
13081	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
13082	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
13083	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
13084	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
13085	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
13086	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
13087	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
13088	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
13089	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
13090	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
13091	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
13092	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
13093	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
13094	) != 0
13095	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
13096	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
13097	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
13098	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
13099	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
13100	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
13101	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
13102	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
13103	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
13104	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
13105	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
13106	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
13107	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
13108	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
13109	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
13110	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
13111	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
13112	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
13113	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
13114	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
13115	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
13116	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
13117	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
13118	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
13119	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
13120	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
13121	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
13122	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
13123	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
13124	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
13125	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
13126	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
13127	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
13128	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
13129	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
13130	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
13131	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
13132	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
13133	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
13134	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
13135	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
13136	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
13137	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
13138	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
13139	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
13140	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
13141	) != 0
13142	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
13143	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
13144	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
13145	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
13146	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
13147	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
13148	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
13149	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
13150	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
13151	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
13152	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
13153	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
13154	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
13155	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
13156	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
13157	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
13158	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
13159	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
13160	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
13161	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
13162	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
13163	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
13164	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
13165	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
13166	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
13167	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
13168	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
13169	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
13170	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
13171	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
13172	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
13173	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
13174	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
13175	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
13176	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
13177	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
13178	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
13179	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
13180	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
13181	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
13182	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
13183	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
13184	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
13185	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
13186	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
13187	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
13188	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
13189	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
13190	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
13191	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
13192	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
13193	) != 0;
13194	}
13195
13196
13197	/**
13198	* Runs the guest using hardware-assisted VMX.
13199	*
13200	* @returns Strict VBox status code (i.e. informational status codes too).
13201	* @param pVCpu The cross context virtual CPU structure.
13202	*/
13203	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
13204	{
13205	AssertPtr(pVCpu);
13206	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13207	Assert(VMMRZCallRing3IsEnabled(pVCpu));
13208	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13209	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
13210
13211	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
13212
13213	VBOXSTRICTRC rcStrict;
13214	uint32_t cLoops = 0;
13215	for (;;)
13216	{
13217	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13218	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
13219	#else
13220	bool const fInNestedGuestMode = false;
13221	#endif
13222	if (!fInNestedGuestMode)
13223	{
13224	if ( !pVCpu->hm.s.fUseDebugLoop
13225	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
13226	&& !DBGFIsStepping(pVCpu)
13227	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
13228	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
13229	else
13230	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
13231	}
13232	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13233	else
13234	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
13235
13236	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
13237	{
13238	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
13239	continue;
13240	}
13241	if (rcStrict == VINF_VMX_VMEXIT)
13242	{
13243	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
13244	continue;
13245	}
13246	#endif
13247	break;
13248	}
13249
13250	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
13251	switch (rcLoop)
13252	{
13253	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
13254	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
13255	}
13256
13257	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
13258	if (RT_FAILURE(rc2))
13259	{
13260	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
13261	rcStrict = rc2;
13262	}
13263	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13264	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
13265	return rcStrict;
13266	}
13267
13268
13269	#ifndef HMVMX_USE_FUNCTION_TABLE
13270	/**
13271	* Handles a guest VM-exit from hardware-assisted VMX execution.
13272	*
13273	* @returns Strict VBox status code (i.e. informational status codes too).
13274	* @param pVCpu The cross context virtual CPU structure.
13275	* @param pVmxTransient The VMX-transient structure.
13276	*/
13277	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13278	{
13279	#ifdef DEBUG_ramshankar
13280	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
13281	do { \
13282	if (a_fSave != 0) \
13283	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
13284	VBOXSTRICTRC rcStrict = a_CallExpr; \
13285	if (a_fSave != 0) \
13286	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
13287	return rcStrict; \
13288	} while (0)
13289	#else
13290	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
13291	#endif
13292	uint32_t const uExitReason = pVmxTransient->uExitReason;
13293	switch (uExitReason)
13294	{
13295	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
13296	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
13297	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
13298	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
13299	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
13300	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
13301	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
13302	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
13303	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
13304	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
13305	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
13306	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
13307	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
13308	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
13309	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
13310	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
13311	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
13312	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
13313	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
13314	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
13315	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
13316	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
13317	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
13318	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
13319	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
13320	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
13321	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
13322	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
13323	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
13324	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
13325	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13326	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
13327	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
13328	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
13329	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
13330	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
13331	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
13332	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
13333	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
13334	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
13335	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
13336	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
13337	#else
13338	case VMX_EXIT_VMCLEAR:
13339	case VMX_EXIT_VMLAUNCH:
13340	case VMX_EXIT_VMPTRLD:
13341	case VMX_EXIT_VMPTRST:
13342	case VMX_EXIT_VMREAD:
13343	case VMX_EXIT_VMRESUME:
13344	case VMX_EXIT_VMWRITE:
13345	case VMX_EXIT_VMXOFF:
13346	case VMX_EXIT_VMXON:
13347	case VMX_EXIT_INVVPID:
13348	case VMX_EXIT_INVEPT:
13349	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
13350	#endif
13351
13352	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
13353	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
13354	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
13355
13356	case VMX_EXIT_INIT_SIGNAL:
13357	case VMX_EXIT_SIPI:
13358	case VMX_EXIT_IO_SMI:
13359	case VMX_EXIT_SMI:
13360	case VMX_EXIT_ERR_MSR_LOAD:
13361	case VMX_EXIT_ERR_MACHINE_CHECK:
13362	case VMX_EXIT_PML_FULL:
13363	case VMX_EXIT_VIRTUALIZED_EOI:
13364	case VMX_EXIT_GDTR_IDTR_ACCESS:
13365	case VMX_EXIT_LDTR_TR_ACCESS:
13366	case VMX_EXIT_APIC_WRITE:
13367	case VMX_EXIT_RDRAND:
13368	case VMX_EXIT_RSM:
13369	case VMX_EXIT_VMFUNC:
13370	case VMX_EXIT_ENCLS:
13371	case VMX_EXIT_RDSEED:
13372	case VMX_EXIT_XSAVES:
13373	case VMX_EXIT_XRSTORS:
13374	case VMX_EXIT_UMWAIT:
13375	case VMX_EXIT_TPAUSE:
13376	default:
13377	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13378	}
13379	#undef VMEXIT_CALL_RET
13380	}
13381	#endif /* !HMVMX_USE_FUNCTION_TABLE */
13382
13383
13384	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13385	/**
13386	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
13387	*
13388	* @returns Strict VBox status code (i.e. informational status codes too).
13389	* @param pVCpu The cross context virtual CPU structure.
13390	* @param pVmxTransient The VMX-transient structure.
13391	*/
13392	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13393	{
13394	uint32_t const uExitReason = pVmxTransient->uExitReason;
13395	switch (uExitReason)
13396	{
13397	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
13398	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
13399	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
13400	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
13401	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
13402
13403	/*
13404	* We shouldn't direct host physical interrupts to the nested-guest.
13405	*/
13406	case VMX_EXIT_EXT_INT:
13407	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
13408
13409	/*
13410	* Instructions that cause VM-exits unconditionally or the condition is
13411	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13412	* happens, it's guaranteed to be a nested-guest VM-exit).
13413	*
13414	* - Provides VM-exit instruction length ONLY.
13415	*/
13416	case VMX_EXIT_CPUID: /* Unconditional. */
13417	case VMX_EXIT_VMCALL:
13418	case VMX_EXIT_GETSEC:
13419	case VMX_EXIT_INVD:
13420	case VMX_EXIT_XSETBV:
13421	case VMX_EXIT_VMLAUNCH:
13422	case VMX_EXIT_VMRESUME:
13423	case VMX_EXIT_VMXOFF:
13424	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
13425	case VMX_EXIT_VMFUNC:
13426	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
13427
13428	/*
13429	* Instructions that cause VM-exits unconditionally or the condition is
13430	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13431	* happens, it's guaranteed to be a nested-guest VM-exit).
13432	*
13433	* - Provides VM-exit instruction length.
13434	* - Provides VM-exit information.
13435	* - Optionally provides Exit qualification.
13436	*
13437	* Since Exit qualification is 0 for all VM-exits where it is not
13438	* applicable, reading and passing it to the guest should produce
13439	* defined behavior.
13440	*
13441	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
13442	*/
13443	case VMX_EXIT_INVEPT: /* Unconditional. */
13444	case VMX_EXIT_INVVPID:
13445	case VMX_EXIT_VMCLEAR:
13446	case VMX_EXIT_VMPTRLD:
13447	case VMX_EXIT_VMPTRST:
13448	case VMX_EXIT_VMXON:
13449	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
13450	case VMX_EXIT_LDTR_TR_ACCESS:
13451	case VMX_EXIT_RDRAND:
13452	case VMX_EXIT_RDSEED:
13453	case VMX_EXIT_XSAVES:
13454	case VMX_EXIT_XRSTORS:
13455	case VMX_EXIT_UMWAIT:
13456	case VMX_EXIT_TPAUSE:
13457	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
13458
13459	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
13460	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
13461	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
13462	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
13463	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
13464	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
13465	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
13466	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
13467	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
13468	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
13469	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
13470	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
13471	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
13472	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
13473	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
13474	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
13475	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
13476	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
13477	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
13478
13479	case VMX_EXIT_PREEMPT_TIMER:
13480	{
13481	/** @todo NSTVMX: Preempt timer. */
13482	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
13483	}
13484
13485	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
13486	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
13487
13488	case VMX_EXIT_VMREAD:
13489	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13490
13491	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13492	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13493
13494	case VMX_EXIT_INIT_SIGNAL:
13495	case VMX_EXIT_SIPI:
13496	case VMX_EXIT_IO_SMI:
13497	case VMX_EXIT_SMI:
13498	case VMX_EXIT_ERR_MSR_LOAD:
13499	case VMX_EXIT_ERR_MACHINE_CHECK:
13500	case VMX_EXIT_PML_FULL:
13501	case VMX_EXIT_RSM:
13502	default:
13503	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13504	}
13505	}
13506	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13507
13508
13509	/** @name VM-exit helpers.
13510	* @{
13511	*/
13512	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13513	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13514	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13515
13516	/** Macro for VM-exits called unexpectedly. */
13517	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
13518	do { \
13519	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
13520	return VERR_VMX_UNEXPECTED_EXIT; \
13521	} while (0)
13522
13523	#ifdef VBOX_STRICT
13524	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13525	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13526	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13527
13528	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13529	do { \
13530	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13531	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13532	} while (0)
13533
13534	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13535	do { \
13536	AssertPtr((a_pVCpu)); \
13537	AssertPtr((a_pVmxTransient)); \
13538	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13539	Assert((a_pVmxTransient)->pVmcsInfo); \
13540	Assert(ASMIntAreEnabled()); \
13541	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13542	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13543	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
13544	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13545	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13546	HMVMX_ASSERT_PREEMPT_CPUID(); \
13547	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13548	} while (0)
13549
13550	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13551	do { \
13552	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13553	Assert((a_pVmxTransient)->fIsNestedGuest); \
13554	} while (0)
13555
13556	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13557	do { \
13558	Log4Func(("\n")); \
13559	} while (0)
13560	#else
13561	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13562	do { \
13563	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13564	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13565	} while (0)
13566
13567	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13568	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13569
13570	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13571	#endif
13572
13573	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13574	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13575	* guests that attempted to execute a VMX instruction. */
13576	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13577	do \
13578	{ \
13579	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13580	if (rcStrictTmp == VINF_SUCCESS) \
13581	{ /* likely */ } \
13582	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13583	{ \
13584	Assert((a_pVCpu)->hm.s.Event.fPending); \
13585	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13586	return VINF_SUCCESS; \
13587	} \
13588	else \
13589	{ \
13590	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13591	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13592	} \
13593	} while (0)
13594
13595	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13596	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13597	do \
13598	{ \
13599	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13600	(a_pGCPtrEffAddr)); \
13601	if (rcStrictTmp == VINF_SUCCESS) \
13602	{ /* likely */ } \
13603	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13604	{ \
13605	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13606	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13607	NOREF(uXcptTmp); \
13608	return VINF_SUCCESS; \
13609	} \
13610	else \
13611	{ \
13612	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13613	return rcStrictTmp; \
13614	} \
13615	} while (0)
13616	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13617
13618
13619	/**
13620	* Advances the guest RIP by the specified number of bytes.
13621	*
13622	* @param pVCpu The cross context virtual CPU structure.
13623	* @param cbInstr Number of bytes to advance the RIP by.
13624	*
13625	* @remarks No-long-jump zone!!!
13626	*/
13627	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13628	{
13629	/* Advance the RIP. */
13630	pVCpu->cpum.GstCtx.rip += cbInstr;
13631	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13632
13633	/* Update interrupt inhibition. */
13634	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13635	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13636	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13637	}
13638
13639
13640	/**
13641	* Advances the guest RIP after reading it from the VMCS.
13642	*
13643	* @returns VBox status code, no informational status codes.
13644	* @param pVCpu The cross context virtual CPU structure.
13645	* @param pVmxTransient The VMX-transient structure.
13646	*
13647	* @remarks No-long-jump zone!!!
13648	*/
13649	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13650	{
13651	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13652	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13653	AssertRCReturn(rc, rc);
13654
13655	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13656	return VINF_SUCCESS;
13657	}
13658
13659
13660	/**
13661	* Handle a condition that occurred while delivering an event through the guest
13662	* IDT.
13663	*
13664	* @returns Strict VBox status code (i.e. informational status codes too).
13665	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13666	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13667	* to continue execution of the guest which will delivery the \#DF.
13668	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13669	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13670	*
13671	* @param pVCpu The cross context virtual CPU structure.
13672	* @param pVmxTransient The VMX-transient structure.
13673	*
13674	* @remarks No-long-jump zone!!!
13675	*/
13676	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13677	{
13678	/* Read the IDT vectoring info. and VM-exit interruption info. */
13679	{
13680	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13681	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13682	AssertRCReturn(rc, rc);
13683	}
13684
13685	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13686	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13687	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13688	{
13689	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13690	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13691
13692	/*
13693	* If the event was a software interrupt (generated with INT n) or a software exception
13694	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13695	* can handle the VM-exit and continue guest execution which will re-execute the
13696	* instruction rather than re-injecting the exception, as that can cause premature
13697	* trips to ring-3 before injection and involve TRPM which currently has no way of
13698	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13699	* the problem).
13700	*/
13701	IEMXCPTRAISE enmRaise;
13702	IEMXCPTRAISEINFO fRaiseInfo;
13703	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13704	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13705	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13706	{
13707	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13708	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13709	}
13710	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13711	{
13712	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13713	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13714	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13715
13716	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13717	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13718	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13719
13720	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13721
13722	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13723	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13724	{
13725	pVmxTransient->fVectoringPF = true;
13726	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13727	}
13728	}
13729	else
13730	{
13731	/*
13732	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13733	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13734	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13735	*/
13736	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13737	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13738	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13739	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13740	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13741	}
13742
13743	/*
13744	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13745	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13746	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13747	* subsequent VM-entry would fail, see @bugref{7445}.
13748	*
13749	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13750	*/
13751	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13752	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13753	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13754	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13755	&& CPUMIsGuestNmiBlocking(pVCpu))
13756	{
13757	CPUMSetGuestNmiBlocking(pVCpu, false);
13758	}
13759
13760	switch (enmRaise)
13761	{
13762	case IEMXCPTRAISE_CURRENT_XCPT:
13763	{
13764	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13765	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13766	Assert(rcStrict == VINF_SUCCESS);
13767	break;
13768	}
13769
13770	case IEMXCPTRAISE_PREV_EVENT:
13771	{
13772	uint32_t u32ErrCode;
13773	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13774	{
13775	int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13776	AssertRCReturn(rc, rc);
13777	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13778	}
13779	else
13780	u32ErrCode = 0;
13781
13782	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13783	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13784	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13785	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13786
13787	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13788	pVCpu->hm.s.Event.u32ErrCode));
13789	Assert(rcStrict == VINF_SUCCESS);
13790	break;
13791	}
13792
13793	case IEMXCPTRAISE_REEXEC_INSTR:
13794	Assert(rcStrict == VINF_SUCCESS);
13795	break;
13796
13797	case IEMXCPTRAISE_DOUBLE_FAULT:
13798	{
13799	/*
13800	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13801	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13802	*/
13803	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13804	{
13805	pVmxTransient->fVectoringDoublePF = true;
13806	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13807	pVCpu->cpum.GstCtx.cr2));
13808	rcStrict = VINF_SUCCESS;
13809	}
13810	else
13811	{
13812	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13813	hmR0VmxSetPendingXcptDF(pVCpu);
13814	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13815	uIdtVector, uExitVector));
13816	rcStrict = VINF_HM_DOUBLE_FAULT;
13817	}
13818	break;
13819	}
13820
13821	case IEMXCPTRAISE_TRIPLE_FAULT:
13822	{
13823	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13824	rcStrict = VINF_EM_RESET;
13825	break;
13826	}
13827
13828	case IEMXCPTRAISE_CPU_HANG:
13829	{
13830	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13831	rcStrict = VERR_EM_GUEST_CPU_HANG;
13832	break;
13833	}
13834
13835	default:
13836	{
13837	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13838	rcStrict = VERR_VMX_IPE_2;
13839	break;
13840	}
13841	}
13842	}
13843	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13844	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13845	&& uExitVector != X86_XCPT_DF
13846	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13847	{
13848	/*
13849	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13850	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13851	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13852	*/
13853	CPUMSetGuestNmiBlocking(pVCpu, true);
13854	Log4Func(("Set NMI blocking. fValid=%RTbool uExitReason=%u\n", VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo),
13855	pVmxTransient->uExitReason));
13856	}
13857
13858	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13859	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13860	return rcStrict;
13861	}
13862
13863
13864	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13865	/**
13866	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13867	* guest attempting to execute a VMX instruction.
13868	*
13869	* @returns Strict VBox status code (i.e. informational status codes too).
13870	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13871	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13872	*
13873	* @param pVCpu The cross context virtual CPU structure.
13874	* @param uExitReason The VM-exit reason.
13875	*
13876	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13877	* @remarks No-long-jump zone!!!
13878	*/
13879	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
13880	{
13881	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13882	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13883
13884	/*
13885	* The physical CPU would have already checked the CPU mode/code segment.
13886	* We shall just assert here for paranoia.
13887	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13888	*/
13889	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13890	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13891	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13892
13893	if (uExitReason == VMX_EXIT_VMXON)
13894	{
13895	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13896
13897	/*
13898	* We check CR4.VMXE because it is required to be always set while in VMX operation
13899	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13900	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13901	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13902	*/
13903	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13904	{
13905	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13906	hmR0VmxSetPendingXcptUD(pVCpu);
13907	return VINF_HM_PENDING_XCPT;
13908	}
13909	}
13910	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13911	{
13912	/*
13913	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13914	* (other than VMXON), we need to raise a #UD.
13915	*/
13916	Log4Func(("Not in VMX root mode -> #UD\n"));
13917	hmR0VmxSetPendingXcptUD(pVCpu);
13918	return VINF_HM_PENDING_XCPT;
13919	}
13920
13921	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13922	return VINF_SUCCESS;
13923	}
13924
13925	/**
13926	* Decodes the memory operand of an instruction that caused a VM-exit.
13927	*
13928	* The Exit qualification field provides the displacement field for memory
13929	* operand instructions, if any.
13930	*
13931	* @returns Strict VBox status code (i.e. informational status codes too).
13932	* @retval VINF_SUCCESS if the operand was successfully decoded.
13933	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13934	* operand.
13935	* @param pVCpu The cross context virtual CPU structure.
13936	* @param uExitInstrInfo The VM-exit instruction information field.
13937	* @param enmMemAccess The memory operand's access type (read or write).
13938	* @param GCPtrDisp The instruction displacement field, if any. For
13939	* RIP-relative addressing pass RIP + displacement here.
13940	* @param pGCPtrMem Where to store the effective destination memory address.
13941	*
13942	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13943	* virtual-8086 mode hence skips those checks while verifying if the
13944	* segment is valid.
13945	*/
13946	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13947	PRTGCPTR pGCPtrMem)
13948	{
13949	Assert(pGCPtrMem);
13950	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13951	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13952	\| CPUMCTX_EXTRN_CR0);
13953
13954	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13955	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13956	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13957
13958	VMXEXITINSTRINFO ExitInstrInfo;
13959	ExitInstrInfo.u = uExitInstrInfo;
13960	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13961	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13962	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13963	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13964	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13965	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13966	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13967	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13968	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13969
13970	/*
13971	* Validate instruction information.
13972	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13973	*/
13974	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13975	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13976	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13977	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13978	AssertLogRelMsgReturn(fIsMemOperand,
13979	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13980
13981	/*
13982	* Compute the complete effective address.
13983	*
13984	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13985	* See AMD spec. 4.5.2 "Segment Registers".
13986	*/
13987	RTGCPTR GCPtrMem = GCPtrDisp;
13988	if (fBaseRegValid)
13989	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13990	if (fIdxRegValid)
13991	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13992
13993	RTGCPTR const GCPtrOff = GCPtrMem;
13994	if ( !fIsLongMode
13995	\|\| iSegReg >= X86_SREG_FS)
13996	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13997	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13998
13999	/*
14000	* Validate effective address.
14001	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
14002	*/
14003	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
14004	Assert(cbAccess > 0);
14005	if (fIsLongMode)
14006	{
14007	if (X86_IS_CANONICAL(GCPtrMem))
14008	{
14009	*pGCPtrMem = GCPtrMem;
14010	return VINF_SUCCESS;
14011	}
14012
14013	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
14014	* "Data Limit Checks in 64-bit Mode". */
14015	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
14016	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14017	return VINF_HM_PENDING_XCPT;
14018	}
14019
14020	/*
14021	* This is a watered down version of iemMemApplySegment().
14022	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
14023	* and segment CPL/DPL checks are skipped.
14024	*/
14025	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
14026	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
14027	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
14028
14029	/* Check if the segment is present and usable. */
14030	if ( pSel->Attr.n.u1Present
14031	&& !pSel->Attr.n.u1Unusable)
14032	{
14033	Assert(pSel->Attr.n.u1DescType);
14034	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
14035	{
14036	/* Check permissions for the data segment. */
14037	if ( enmMemAccess == VMXMEMACCESS_WRITE
14038	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
14039	{
14040	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14041	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
14042	return VINF_HM_PENDING_XCPT;
14043	}
14044
14045	/* Check limits if it's a normal data segment. */
14046	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
14047	{
14048	if ( GCPtrFirst32 > pSel->u32Limit
14049	\|\| GCPtrLast32 > pSel->u32Limit)
14050	{
14051	Log4Func(("Data segment limit exceeded. "
14052	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
14053	GCPtrLast32, pSel->u32Limit));
14054	if (iSegReg == X86_SREG_SS)
14055	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14056	else
14057	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14058	return VINF_HM_PENDING_XCPT;
14059	}
14060	}
14061	else
14062	{
14063	/* Check limits if it's an expand-down data segment.
14064	Note! The upper boundary is defined by the B bit, not the G bit! */
14065	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
14066	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
14067	{
14068	Log4Func(("Expand-down data segment limit exceeded. "
14069	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
14070	GCPtrLast32, pSel->u32Limit));
14071	if (iSegReg == X86_SREG_SS)
14072	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14073	else
14074	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14075	return VINF_HM_PENDING_XCPT;
14076	}
14077	}
14078	}
14079	else
14080	{
14081	/* Check permissions for the code segment. */
14082	if ( enmMemAccess == VMXMEMACCESS_WRITE
14083	\|\| ( enmMemAccess == VMXMEMACCESS_READ
14084	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
14085	{
14086	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
14087	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
14088	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14089	return VINF_HM_PENDING_XCPT;
14090	}
14091
14092	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
14093	if ( GCPtrFirst32 > pSel->u32Limit
14094	\|\| GCPtrLast32 > pSel->u32Limit)
14095	{
14096	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
14097	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
14098	if (iSegReg == X86_SREG_SS)
14099	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14100	else
14101	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14102	return VINF_HM_PENDING_XCPT;
14103	}
14104	}
14105	}
14106	else
14107	{
14108	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14109	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14110	return VINF_HM_PENDING_XCPT;
14111	}
14112
14113	*pGCPtrMem = GCPtrMem;
14114	return VINF_SUCCESS;
14115	}
14116	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
14117
14118
14119	/**
14120	* VM-exit helper for LMSW.
14121	*/
14122	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
14123	{
14124	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14125	AssertRCReturn(rc, rc);
14126
14127	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
14128	AssertMsg( rcStrict == VINF_SUCCESS
14129	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14130
14131	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14132	if (rcStrict == VINF_IEM_RAISED_XCPT)
14133	{
14134	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14135	rcStrict = VINF_SUCCESS;
14136	}
14137
14138	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14139	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14140	return rcStrict;
14141	}
14142
14143
14144	/**
14145	* VM-exit helper for CLTS.
14146	*/
14147	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
14148	{
14149	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14150	AssertRCReturn(rc, rc);
14151
14152	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
14153	AssertMsg( rcStrict == VINF_SUCCESS
14154	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14155
14156	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14157	if (rcStrict == VINF_IEM_RAISED_XCPT)
14158	{
14159	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14160	rcStrict = VINF_SUCCESS;
14161	}
14162
14163	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14164	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14165	return rcStrict;
14166	}
14167
14168
14169	/**
14170	* VM-exit helper for MOV from CRx (CRx read).
14171	*/
14172	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14173	{
14174	Assert(iCrReg < 16);
14175	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14176
14177	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14178	AssertRCReturn(rc, rc);
14179
14180	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
14181	AssertMsg( rcStrict == VINF_SUCCESS
14182	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14183
14184	if (iGReg == X86_GREG_xSP)
14185	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14186	else
14187	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14188	#ifdef VBOX_WITH_STATISTICS
14189	switch (iCrReg)
14190	{
14191	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14192	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14193	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14194	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14195	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14196	}
14197	#endif
14198	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
14199	return rcStrict;
14200	}
14201
14202
14203	/**
14204	* VM-exit helper for MOV to CRx (CRx write).
14205	*/
14206	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14207	{
14208	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14209	AssertRCReturn(rc, rc);
14210
14211	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
14212	AssertMsg( rcStrict == VINF_SUCCESS
14213	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14214	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14215
14216	switch (iCrReg)
14217	{
14218	case 0:
14219	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14220	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14221	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14222	break;
14223
14224	case 2:
14225	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14226	/* Nothing to do here, CR2 it's not part of the VMCS. */
14227	break;
14228
14229	case 3:
14230	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14232	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14233	break;
14234
14235	case 4:
14236	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14237	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14238	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14239	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14240	break;
14241
14242	case 8:
14243	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14244	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14246	break;
14247
14248	default:
14249	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
14250	break;
14251	}
14252
14253	if (rcStrict == VINF_IEM_RAISED_XCPT)
14254	{
14255	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14256	rcStrict = VINF_SUCCESS;
14257	}
14258	return rcStrict;
14259	}
14260
14261
14262	/**
14263	* VM-exit exception handler for \#PF (Page-fault exception).
14264	*/
14265	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14266	{
14267	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14268	PVM pVM = pVCpu->CTX_SUFF(pVM);
14269	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14270	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14271	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14272	AssertRCReturn(rc, rc);
14273
14274	if (!pVM->hm.s.fNestedPaging)
14275	{ /* likely */ }
14276	else
14277	{
14278	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
14279	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
14280	#endif
14281	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
14282	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
14283	{
14284	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14285	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
14286	}
14287	else
14288	{
14289	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14290	hmR0VmxSetPendingXcptDF(pVCpu);
14291	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
14292	}
14293	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14294	return rc;
14295	}
14296
14297	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
14298	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
14299	if (pVmxTransient->fVectoringPF)
14300	{
14301	Assert(pVCpu->hm.s.Event.fPending);
14302	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14303	}
14304
14305	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14306	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14307	AssertRCReturn(rc, rc);
14308
14309	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
14310	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
14311
14312	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
14313	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
14314
14315	Log4Func(("#PF: rc=%Rrc\n", rc));
14316	if (rc == VINF_SUCCESS)
14317	{
14318	/*
14319	* This is typically a shadow page table sync or a MMIO instruction. But we may have
14320	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
14321	*/
14322	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14323	TRPMResetTrap(pVCpu);
14324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
14325	return rc;
14326	}
14327
14328	if (rc == VINF_EM_RAW_GUEST_TRAP)
14329	{
14330	if (!pVmxTransient->fVectoringDoublePF)
14331	{
14332	/* It's a guest page fault and needs to be reflected to the guest. */
14333	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
14334	TRPMResetTrap(pVCpu);
14335	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
14336	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14337	uGstErrorCode, pVmxTransient->uExitQual);
14338	}
14339	else
14340	{
14341	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14342	TRPMResetTrap(pVCpu);
14343	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14344	hmR0VmxSetPendingXcptDF(pVCpu);
14345	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
14346	}
14347
14348	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14349	return VINF_SUCCESS;
14350	}
14351
14352	TRPMResetTrap(pVCpu);
14353	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14354	return rc;
14355	}
14356
14357
14358	/**
14359	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14360	*/
14361	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14362	{
14363	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14364	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14365
14366	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14367	AssertRCReturn(rc, rc);
14368
14369	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14370	{
14371	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14372	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14373
14374	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14375	* provides VM-exit instruction length. If this causes problem later,
14376	* disassemble the instruction like it's done on AMD-V. */
14377	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14378	AssertRCReturn(rc2, rc2);
14379	return rc;
14380	}
14381
14382	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14383	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14384	return rc;
14385	}
14386
14387
14388	/**
14389	* VM-exit exception handler for \#BP (Breakpoint exception).
14390	*/
14391	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14392	{
14393	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14394	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14395
14396	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14397	AssertRCReturn(rc, rc);
14398
14399	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14400	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14401	if (rc == VINF_EM_RAW_GUEST_TRAP)
14402	{
14403	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14404	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14405	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14406	AssertRCReturn(rc, rc);
14407
14408	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14409	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14410	}
14411
14412	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14413	return rc;
14414	}
14415
14416
14417	/**
14418	* VM-exit exception handler for \#AC (alignment check exception).
14419	*/
14420	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14421	{
14422	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14423	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
14424
14425	/*
14426	* Re-inject it. We'll detect any nesting before getting here.
14427	*/
14428	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14429	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14430	AssertRCReturn(rc, rc);
14431	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14432
14433	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14434	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14435	return VINF_SUCCESS;
14436	}
14437
14438
14439	/**
14440	* VM-exit exception handler for \#DB (Debug exception).
14441	*/
14442	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14443	{
14444	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14445	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14446
14447	/*
14448	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
14449	*/
14450	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14451
14452	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14453	uint64_t const uDR6 = X86_DR6_INIT_VAL
14454	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14455	\| X86_DR6_BD \| X86_DR6_BS));
14456
14457	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14458	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14459	Log6Func(("rc=%Rrc\n", rc));
14460	if (rc == VINF_EM_RAW_GUEST_TRAP)
14461	{
14462	/*
14463	* The exception was for the guest. Update DR6, DR7.GD and
14464	* IA32_DEBUGCTL.LBR before forwarding it.
14465	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14466	*/
14467	VMMRZCallRing3Disable(pVCpu);
14468	HM_DISABLE_PREEMPT(pVCpu);
14469
14470	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14471	pCtx->dr[6] \|= uDR6;
14472	if (CPUMIsGuestDebugStateActive(pVCpu))
14473	ASMSetDR6(pCtx->dr[6]);
14474
14475	HM_RESTORE_PREEMPT();
14476	VMMRZCallRing3Enable(pVCpu);
14477
14478	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14479	AssertRCReturn(rc, rc);
14480
14481	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14482	pCtx->dr[7] &= ~X86_DR7_GD;
14483
14484	/* Paranoia. */
14485	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14486	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14487
14488	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14489	AssertRCReturn(rc, rc);
14490
14491	/*
14492	* Raise #DB in the guest.
14493	*
14494	* It is important to reflect exactly what the VM-exit gave us (preserving the
14495	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14496	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14497	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14498	*
14499	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14500	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14501	*/
14502	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14503	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14504	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14505	AssertRCReturn(rc, rc);
14506	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14507	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14508	return VINF_SUCCESS;
14509	}
14510
14511	/*
14512	* Not a guest trap, must be a hypervisor related debug event then.
14513	* Update DR6 in case someone is interested in it.
14514	*/
14515	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14516	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14517	CPUMSetHyperDR6(pVCpu, uDR6);
14518
14519	return rc;
14520	}
14521
14522
14523	/**
14524	* Hacks its way around the lovely mesa driver's backdoor accesses.
14525	*
14526	* @sa hmR0SvmHandleMesaDrvGp.
14527	*/
14528	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14529	{
14530	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14531	RT_NOREF(pCtx);
14532
14533	/* For now we'll just skip the instruction. */
14534	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14535	}
14536
14537
14538	/**
14539	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14540	* backdoor logging w/o checking what it is running inside.
14541	*
14542	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14543	* backdoor port and magic numbers loaded in registers.
14544	*
14545	* @returns true if it is, false if it isn't.
14546	* @sa hmR0SvmIsMesaDrvGp.
14547	*/
14548	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14549	{
14550	/* 0xed: IN eAX,dx */
14551	uint8_t abInstr[1];
14552	if (pVmxTransient->cbInstr != sizeof(abInstr))
14553	return false;
14554
14555	/* Check that it is #GP(0). */
14556	if (pVmxTransient->uExitIntErrorCode != 0)
14557	return false;
14558
14559	/* Check magic and port. */
14560	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14561	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14562	if (pCtx->rax != UINT32_C(0x564d5868))
14563	return false;
14564	if (pCtx->dx != UINT32_C(0x5658))
14565	return false;
14566
14567	/* Flat ring-3 CS. */
14568	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14569	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14570	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14571	if (pCtx->cs.Attr.n.u2Dpl != 3)
14572	return false;
14573	if (pCtx->cs.u64Base != 0)
14574	return false;
14575
14576	/* Check opcode. */
14577	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14578	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14579	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14580	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14581	if (RT_FAILURE(rc))
14582	return false;
14583	if (abInstr[0] != 0xed)
14584	return false;
14585
14586	return true;
14587	}
14588
14589	/**
14590	* VM-exit exception handler for \#GP (General-protection exception).
14591	*
14592	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14593	*/
14594	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14595	{
14596	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14597	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14598
14599	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14600	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14601	if (pVmcsInfo->RealMode.fRealOnV86Active)
14602	{ /* likely */ }
14603	else
14604	{
14605	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14606	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14607	#endif
14608	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14609	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14610	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14611	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14612	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14613	AssertRCReturn(rc, rc);
14614	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14615	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14616
14617	if ( pVmxTransient->fIsNestedGuest
14618	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14619	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14620	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14621	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14622	else
14623	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14624	return rc;
14625	}
14626
14627	Assert(CPUMIsGuestInRealModeEx(pCtx));
14628	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14629	Assert(!pVmxTransient->fIsNestedGuest);
14630
14631	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14632	AssertRCReturn(rc, rc);
14633
14634	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14635	if (rcStrict == VINF_SUCCESS)
14636	{
14637	if (!CPUMIsGuestInRealModeEx(pCtx))
14638	{
14639	/*
14640	* The guest is no longer in real-mode, check if we can continue executing the
14641	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14642	*/
14643	pVmcsInfo->RealMode.fRealOnV86Active = false;
14644	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14645	{
14646	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14647	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14648	}
14649	else
14650	{
14651	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14652	rcStrict = VINF_EM_RESCHEDULE;
14653	}
14654	}
14655	else
14656	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14657	}
14658	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14659	{
14660	rcStrict = VINF_SUCCESS;
14661	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14662	}
14663	return VBOXSTRICTRC_VAL(rcStrict);
14664	}
14665
14666
14667	/**
14668	* VM-exit exception handler wrapper for generic exceptions.
14669	*
14670	* This simply re-injects the exception back into the VM without any special
14671	* processing.
14672	*/
14673	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14674	{
14675	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14676
14677	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14678	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14679	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14680	("uVector=%#x u32XcptBitmap=%#X32\n",
14681	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14682	NOREF(pVmcsInfo);
14683	#endif
14684
14685	/*
14686	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14687	* would have been handled while checking exits due to event delivery.
14688	*/
14689	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14690	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14691	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14692	AssertRCReturn(rc, rc);
14693
14694	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14695
14696	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14697	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14698	AssertRCReturn(rc, rc);
14699	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14700	#endif
14701
14702	#ifdef VBOX_WITH_STATISTICS
14703	switch (uVector)
14704	{
14705	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14706	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14707	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14708	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14709	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14710	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14711	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14712	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14713	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14714	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14715	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14716	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14717	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14718	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14719	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14720	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14721	default:
14722	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14723	break;
14724	}
14725	#endif
14726
14727	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14728	Assert( uVector != X86_XCPT_PF
14729	\|\| VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo) != VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
14730	NOREF(uVector);
14731
14732	/* Re-inject the original exception into the guest. */
14733	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14734	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14735	return VINF_SUCCESS;
14736	}
14737	/** @} */
14738
14739
14740	/** @name VM-exit handlers.
14741	* @{
14742	*/
14743	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14744	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14745	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14746
14747	/**
14748	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14749	*/
14750	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14751	{
14752	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14753	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14754	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14755	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14756	return VINF_SUCCESS;
14757	return VINF_EM_RAW_INTERRUPT;
14758	}
14759
14760
14761	/**
14762	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14763	* VM-exit.
14764	*/
14765	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14766	{
14767	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14768	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14769
14770	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14771	AssertRCReturn(rc, rc);
14772
14773	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14774	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14775
14776	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14777	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14778	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14779	NOREF(pVmcsInfo);
14780
14781	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
14782	{
14783	/*
14784	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14785	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14786	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14787	*
14788	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14789	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
14790	*/
14791	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14792	return hmR0VmxExitHostNmi(pVCpu);
14793	}
14794
14795	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14796	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14797	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
14798	{ /* likely */ }
14799	else
14800	{
14801	if (rcStrict == VINF_HM_DOUBLE_FAULT)
14802	rcStrict = VINF_SUCCESS;
14803	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14804	return rcStrict;
14805	}
14806
14807	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
14808	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
14809	switch (uIntType)
14810	{
14811	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
14812	Assert(uVector == X86_XCPT_DB);
14813	RT_FALL_THRU();
14814	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
14815	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14816	RT_FALL_THRU();
14817	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14818	{
14819	/*
14820	* If there's any exception caused as a result of event injection, the resulting
14821	* secondary/final execption will be pending, we shall continue guest execution
14822	* after injecting the event. The page-fault case is complicated and we manually
14823	* handle any currently pending event in hmR0VmxExitXcptPF.
14824	*/
14825	if (!pVCpu->hm.s.Event.fPending)
14826	{ /* likely */ }
14827	else if (uVector != X86_XCPT_PF)
14828	{
14829	rcStrict = VINF_SUCCESS;
14830	break;
14831	}
14832
14833	switch (uVector)
14834	{
14835	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
14836	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
14837	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
14838	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
14839	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
14840	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
14841	default:
14842	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient);
14843	break;
14844	}
14845	break;
14846	}
14847
14848	default:
14849	{
14850	pVCpu->hm.s.u32HMError = uExitIntInfo;
14851	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14852	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
14853	break;
14854	}
14855	}
14856	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14857	return rcStrict;
14858	}
14859
14860
14861	/**
14862	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14863	*/
14864	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14865	{
14866	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14867
14868	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14869	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14870	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14871	AssertRCReturn(rc, rc);
14872
14873	/* Evaluate and deliver pending events and resume guest execution. */
14874	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14875	return VINF_SUCCESS;
14876	}
14877
14878
14879	/**
14880	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14881	*/
14882	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14883	{
14884	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14885
14886	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14887	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14888	{
14889	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14890	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14891	}
14892
14893	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14894
14895	/*
14896	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14897	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14898	*/
14899	uint32_t fIntrState;
14900	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14901	AssertRCReturn(rc, rc);
14902	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14903	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14904	{
14905	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14906	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14907
14908	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14909	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14910	AssertRCReturn(rc, rc);
14911	}
14912
14913	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14914	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14915	AssertRCReturn(rc, rc);
14916
14917	/* Evaluate and deliver pending events and resume guest execution. */
14918	return VINF_SUCCESS;
14919	}
14920
14921
14922	/**
14923	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14924	*/
14925	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14926	{
14927	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14928	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14929	}
14930
14931
14932	/**
14933	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14934	*/
14935	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14936	{
14937	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14938	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14939	}
14940
14941
14942	/**
14943	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14944	*/
14945	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14946	{
14947	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14948
14949	/*
14950	* Get the state we need and update the exit history entry.
14951	*/
14952	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14953	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14954	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14955	AssertRCReturn(rc, rc);
14956
14957	VBOXSTRICTRC rcStrict;
14958	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14959	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14960	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14961	if (!pExitRec)
14962	{
14963	/*
14964	* Regular CPUID instruction execution.
14965	*/
14966	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
14967	if (rcStrict == VINF_SUCCESS)
14968	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14969	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14970	{
14971	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14972	rcStrict = VINF_SUCCESS;
14973	}
14974	}
14975	else
14976	{
14977	/*
14978	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14979	*/
14980	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14981	AssertRCReturn(rc2, rc2);
14982
14983	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14984	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14985
14986	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14987	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14988
14989	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14990	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14991	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14992	}
14993	return rcStrict;
14994	}
14995
14996
14997	/**
14998	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14999	*/
15000	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15001	{
15002	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15003
15004	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15005	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
15006	AssertRCReturn(rc, rc);
15007
15008	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
15009	return VINF_EM_RAW_EMULATE_INSTR;
15010
15011	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
15012	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15013	}
15014
15015
15016	/**
15017	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
15018	*/
15019	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15020	{
15021	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15022
15023	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15024	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15025	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15026	AssertRCReturn(rc, rc);
15027
15028	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
15029	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15030	{
15031	/* If we get a spurious VM-exit when TSC offsetting is enabled,
15032	we must reset offsetting on VM-entry. See @bugref{6634}. */
15033	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
15034	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15035	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15036	}
15037	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15038	{
15039	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15040	rcStrict = VINF_SUCCESS;
15041	}
15042	return rcStrict;
15043	}
15044
15045
15046	/**
15047	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
15048	*/
15049	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15050	{
15051	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15052
15053	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15054	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
15055	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15056	AssertRCReturn(rc, rc);
15057
15058	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
15059	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15060	{
15061	/* If we get a spurious VM-exit when TSC offsetting is enabled,
15062	we must reset offsetting on VM-reentry. See @bugref{6634}. */
15063	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
15064	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15065	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15066	}
15067	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15068	{
15069	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15070	rcStrict = VINF_SUCCESS;
15071	}
15072	return rcStrict;
15073	}
15074
15075
15076	/**
15077	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
15078	*/
15079	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15080	{
15081	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15082
15083	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15084	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
15085	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
15086	AssertRCReturn(rc, rc);
15087
15088	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15089	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15090	if (RT_LIKELY(rc == VINF_SUCCESS))
15091	{
15092	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15093	Assert(pVmxTransient->cbInstr == 2);
15094	}
15095	else
15096	{
15097	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
15098	rc = VERR_EM_INTERPRETER;
15099	}
15100	return rc;
15101	}
15102
15103
15104	/**
15105	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
15106	*/
15107	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15108	{
15109	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15110
15111	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
15112	if (EMAreHypercallInstructionsEnabled(pVCpu))
15113	{
15114	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15115	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
15116	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
15117	AssertRCReturn(rc, rc);
15118
15119	/* Perform the hypercall. */
15120	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
15121	if (rcStrict == VINF_SUCCESS)
15122	{
15123	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15124	AssertRCReturn(rc, rc);
15125	}
15126	else
15127	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
15128	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
15129	\|\| RT_FAILURE(rcStrict));
15130
15131	/* If the hypercall changes anything other than guest's general-purpose registers,
15132	we would need to reload the guest changed bits here before VM-entry. */
15133	}
15134	else
15135	Log4Func(("Hypercalls not enabled\n"));
15136
15137	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
15138	if (RT_FAILURE(rcStrict))
15139	{
15140	hmR0VmxSetPendingXcptUD(pVCpu);
15141	rcStrict = VINF_SUCCESS;
15142	}
15143
15144	return rcStrict;
15145	}
15146
15147
15148	/**
15149	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
15150	*/
15151	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15152	{
15153	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15154	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
15155
15156	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15157	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15158	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15159	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15160	AssertRCReturn(rc, rc);
15161
15162	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
15163
15164	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
15165	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15166	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15167	{
15168	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15169	rcStrict = VINF_SUCCESS;
15170	}
15171	else
15172	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
15173	VBOXSTRICTRC_VAL(rcStrict)));
15174	return rcStrict;
15175	}
15176
15177
15178	/**
15179	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
15180	*/
15181	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15182	{
15183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15184
15185	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15186	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
15187	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15188	AssertRCReturn(rc, rc);
15189
15190	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
15191	if (rcStrict == VINF_SUCCESS)
15192	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15193	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15194	{
15195	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15196	rcStrict = VINF_SUCCESS;
15197	}
15198
15199	return rcStrict;
15200	}
15201
15202
15203	/**
15204	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
15205	*/
15206	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15207	{
15208	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15209
15210	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15211	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15212	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15213	AssertRCReturn(rc, rc);
15214
15215	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
15216	if (RT_SUCCESS(rcStrict))
15217	{
15218	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15219	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
15220	rcStrict = VINF_SUCCESS;
15221	}
15222
15223	return rcStrict;
15224	}
15225
15226
15227	/**
15228	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
15229	* VM-exit.
15230	*/
15231	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15232	{
15233	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15234	return VINF_EM_RESET;
15235	}
15236
15237
15238	/**
15239	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
15240	*/
15241	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15242	{
15243	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15244
15245	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15246	AssertRCReturn(rc, rc);
15247
15248	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
15249	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
15250	rc = VINF_SUCCESS;
15251	else
15252	rc = VINF_EM_HALT;
15253
15254	if (rc != VINF_SUCCESS)
15255	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
15256	return rc;
15257	}
15258
15259
15260	/**
15261	* VM-exit handler for instructions that result in a \#UD exception delivered to
15262	* the guest.
15263	*/
15264	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15265	{
15266	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15267	hmR0VmxSetPendingXcptUD(pVCpu);
15268	return VINF_SUCCESS;
15269	}
15270
15271
15272	/**
15273	* VM-exit handler for expiry of the VMX-preemption timer.
15274	*/
15275	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15276	{
15277	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15278
15279	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
15280	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15281
15282	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
15283	PVM pVM = pVCpu->CTX_SUFF(pVM);
15284	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
15285	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
15286	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
15287	}
15288
15289
15290	/**
15291	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
15292	*/
15293	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15294	{
15295	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15296
15297	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15298	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15299	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
15300	AssertRCReturn(rc, rc);
15301
15302	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
15303	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15304	: HM_CHANGED_RAISED_XCPT_MASK);
15305
15306	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15307	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
15308
15309	return rcStrict;
15310	}
15311
15312
15313	/**
15314	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
15315	*/
15316	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15317	{
15318	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15319	/** @todo Use VM-exit instruction information. */
15320	return VERR_EM_INTERPRETER;
15321	}
15322
15323
15324	/**
15325	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
15326	* VM-exit.
15327	*/
15328	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15329	{
15330	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15331	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15332	AssertRCReturn(rc, rc);
15333
15334	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
15335	if (RT_FAILURE(rc))
15336	return rc;
15337
15338	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
15339	NOREF(uInvalidReason);
15340
15341	#ifdef VBOX_STRICT
15342	uint32_t fIntrState;
15343	RTHCUINTREG uHCReg;
15344	uint64_t u64Val;
15345	uint32_t u32Val;
15346	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
15347	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
15348	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
15349	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
15350	AssertRCReturn(rc, rc);
15351
15352	Log4(("uInvalidReason %u\n", uInvalidReason));
15353	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
15354	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
15355	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
15356	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
15357
15358	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
15359	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
15360	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
15361	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
15362	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
15363	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15364	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
15365	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
15366	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
15367	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15368	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
15369	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
15370
15371	hmR0DumpRegs(pVCpu);
15372	#endif
15373
15374	return VERR_VMX_INVALID_GUEST_STATE;
15375	}
15376
15377	/**
15378	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
15379	*/
15380	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15381	{
15382	/*
15383	* Cummulative notes of all recognized but unexpected VM-exits.
15384	*
15385	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
15386	* nested-paging is used.
15387	*
15388	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
15389	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
15390	* this function (and thereby stop VM execution) for handling such instructions.
15391	*
15392	*
15393	* VMX_EXIT_INIT_SIGNAL:
15394	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
15395	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
15396	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
15397	*
15398	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
15399	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
15400	* See Intel spec. "23.8 Restrictions on VMX operation".
15401	*
15402	* VMX_EXIT_SIPI:
15403	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
15404	* activity state is used. We don't make use of it as our guests don't have direct
15405	* access to the host local APIC.
15406	*
15407	* See Intel spec. 25.3 "Other Causes of VM-exits".
15408	*
15409	* VMX_EXIT_IO_SMI:
15410	* VMX_EXIT_SMI:
15411	* This can only happen if we support dual-monitor treatment of SMI, which can be
15412	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
15413	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
15414	* VMX root mode or receive an SMI. If we get here, something funny is going on.
15415	*
15416	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
15417	* See Intel spec. 25.3 "Other Causes of VM-Exits"
15418	*
15419	* VMX_EXIT_ERR_MSR_LOAD:
15420	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
15421	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
15422	* execution.
15423	*
15424	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
15425	*
15426	* VMX_EXIT_ERR_MACHINE_CHECK:
15427	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
15428	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
15429	* #MC exception abort class exception is raised. We thus cannot assume a
15430	* reasonable chance of continuing any sort of execution and we bail.
15431	*
15432	* See Intel spec. 15.1 "Machine-check Architecture".
15433	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
15434	*
15435	* VMX_EXIT_PML_FULL:
15436	* VMX_EXIT_VIRTUALIZED_EOI:
15437	* VMX_EXIT_APIC_WRITE:
15438	* We do not currently support any of these features and thus they are all unexpected
15439	* VM-exits.
15440	*
15441	* VMX_EXIT_GDTR_IDTR_ACCESS:
15442	* VMX_EXIT_LDTR_TR_ACCESS:
15443	* VMX_EXIT_RDRAND:
15444	* VMX_EXIT_RSM:
15445	* VMX_EXIT_VMFUNC:
15446	* VMX_EXIT_ENCLS:
15447	* VMX_EXIT_RDSEED:
15448	* VMX_EXIT_XSAVES:
15449	* VMX_EXIT_XRSTORS:
15450	* VMX_EXIT_UMWAIT:
15451	* VMX_EXIT_TPAUSE:
15452	* These VM-exits are -not- caused unconditionally by execution of the corresponding
15453	* instruction. Any VM-exit for these instructions indicate a hardware problem,
15454	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
15455	*
15456	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
15457	*/
15458	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15459	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
15460	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15461	}
15462
15463
15464	/**
15465	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
15466	*/
15467	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15468	{
15469	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15470
15471	/** @todo Optimize this: We currently drag in in the whole MSR state
15472	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15473	* MSRs required. That would require changes to IEM and possibly CPUM too.
15474	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15475	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15476	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15477	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15478	switch (idMsr)
15479	{
15480	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15481	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15482	}
15483
15484	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15485	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15486	AssertRCReturn(rc, rc);
15487
15488	Log4Func(("ecx=%#RX32\n", idMsr));
15489
15490	#ifdef VBOX_STRICT
15491	Assert(!pVmxTransient->fIsNestedGuest);
15492	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15493	{
15494	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15495	&& idMsr != MSR_K6_EFER)
15496	{
15497	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15498	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15499	}
15500	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15501	{
15502	Assert(pVmcsInfo->pvMsrBitmap);
15503	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15504	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15505	{
15506	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15507	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15508	}
15509	}
15510	}
15511	#endif
15512
15513	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
15514	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15515	if (rcStrict == VINF_SUCCESS)
15516	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15517	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
15518	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15519	{
15520	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15521	rcStrict = VINF_SUCCESS;
15522	}
15523	else
15524	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15525
15526	return rcStrict;
15527	}
15528
15529
15530	/**
15531	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15532	*/
15533	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15534	{
15535	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15536
15537	/** @todo Optimize this: We currently drag in in the whole MSR state
15538	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15539	* MSRs required. That would require changes to IEM and possibly CPUM too.
15540	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15541	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15542	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15543
15544	/*
15545	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15546	* Although we don't need to fetch the base as it will be overwritten shortly, while
15547	* loading guest-state we would also load the entire segment register including limit
15548	* and attributes and thus we need to load them here.
15549	*/
15550	switch (idMsr)
15551	{
15552	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15553	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15554	}
15555
15556	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15557	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15558	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15559	AssertRCReturn(rc, rc);
15560
15561	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15562
15563	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
15564	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15565
15566	if (rcStrict == VINF_SUCCESS)
15567	{
15568	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15569
15570	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15571	if ( idMsr == MSR_IA32_APICBASE
15572	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15573	&& idMsr <= MSR_IA32_X2APIC_END))
15574	{
15575	/*
15576	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15577	* When full APIC register virtualization is implemented we'll have to make
15578	* sure APIC state is saved from the VMCS before IEM changes it.
15579	*/
15580	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15581	}
15582	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15583	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15584	else if (idMsr == MSR_K6_EFER)
15585	{
15586	/*
15587	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15588	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15589	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15590	*/
15591	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15592	}
15593
15594	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15595	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15596	{
15597	switch (idMsr)
15598	{
15599	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15600	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15601	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15602	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15603	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15604	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15605	default:
15606	{
15607	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15608	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15609	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15610	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15611	break;
15612	}
15613	}
15614	}
15615	#ifdef VBOX_STRICT
15616	else
15617	{
15618	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15619	switch (idMsr)
15620	{
15621	case MSR_IA32_SYSENTER_CS:
15622	case MSR_IA32_SYSENTER_EIP:
15623	case MSR_IA32_SYSENTER_ESP:
15624	case MSR_K8_FS_BASE:
15625	case MSR_K8_GS_BASE:
15626	{
15627	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15628	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15629	}
15630
15631	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15632	default:
15633	{
15634	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15635	{
15636	/* EFER MSR writes are always intercepted. */
15637	if (idMsr != MSR_K6_EFER)
15638	{
15639	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15640	idMsr));
15641	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15642	}
15643	}
15644
15645	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15646	{
15647	Assert(pVmcsInfo->pvMsrBitmap);
15648	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15649	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15650	{
15651	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15652	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15653	}
15654	}
15655	break;
15656	}
15657	}
15658	}
15659	#endif /* VBOX_STRICT */
15660	}
15661	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15662	{
15663	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15664	rcStrict = VINF_SUCCESS;
15665	}
15666	else
15667	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15668
15669	return rcStrict;
15670	}
15671
15672
15673	/**
15674	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15675	*/
15676	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15677	{
15678	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15679
15680	/** @todo The guest has likely hit a contended spinlock. We might want to
15681	* poke a schedule different guest VCPU. */
15682	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15683	if (RT_SUCCESS(rc))
15684	return VINF_EM_RAW_INTERRUPT;
15685
15686	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15687	return rc;
15688	}
15689
15690
15691	/**
15692	* VM-exit handler for when the TPR value is lowered below the specified
15693	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15694	*/
15695	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15696	{
15697	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15698	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15699
15700	/*
15701	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15702	* We'll re-evaluate pending interrupts and inject them before the next VM
15703	* entry so we can just continue execution here.
15704	*/
15705	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15706	return VINF_SUCCESS;
15707	}
15708
15709
15710	/**
15711	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15712	* VM-exit.
15713	*
15714	* @retval VINF_SUCCESS when guest execution can continue.
15715	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15716	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15717	* incompatible guest state for VMX execution (real-on-v86 case).
15718	*/
15719	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15720	{
15721	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15722	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15723
15724	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15725	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15726	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15727	AssertRCReturn(rc, rc);
15728
15729	VBOXSTRICTRC rcStrict;
15730	PVM pVM = pVCpu->CTX_SUFF(pVM);
15731	uint64_t const uExitQual = pVmxTransient->uExitQual;
15732	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15733	switch (uAccessType)
15734	{
15735	/*
15736	* MOV to CRx.
15737	*/
15738	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15739	{
15740	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15741	AssertRCReturn(rc, rc);
15742
15743	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15744	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15745	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15746	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15747
15748	/*
15749	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15750	* - When nested paging isn't used.
15751	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15752	* - We are executing in the VM debug loop.
15753	*/
15754	Assert( iCrReg != 3
15755	\|\| !pVM->hm.s.fNestedPaging
15756	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15757	\|\| pVCpu->hm.s.fUsingDebugLoop);
15758
15759	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15760	Assert( iCrReg != 8
15761	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15762
15763	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15764	AssertMsg( rcStrict == VINF_SUCCESS
15765	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15766
15767	/*
15768	* This is a kludge for handling switches back to real mode when we try to use
15769	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15770	* deal with special selector values, so we have to return to ring-3 and run
15771	* there till the selector values are V86 mode compatible.
15772	*
15773	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15774	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15775	* this function.
15776	*/
15777	if ( iCrReg == 0
15778	&& rcStrict == VINF_SUCCESS
15779	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15780	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15781	&& (uOldCr0 & X86_CR0_PE)
15782	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15783	{
15784	/** @todo Check selectors rather than returning all the time. */
15785	Assert(!pVmxTransient->fIsNestedGuest);
15786	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15787	rcStrict = VINF_EM_RESCHEDULE_REM;
15788	}
15789	break;
15790	}
15791
15792	/*
15793	* MOV from CRx.
15794	*/
15795	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15796	{
15797	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15798	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15799
15800	/*
15801	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15802	* - When nested paging isn't used.
15803	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15804	* - We are executing in the VM debug loop.
15805	*/
15806	Assert( iCrReg != 3
15807	\|\| !pVM->hm.s.fNestedPaging
15808	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15809	\|\| pVCpu->hm.s.fUsingDebugLoop);
15810
15811	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15812	Assert( iCrReg != 8
15813	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15814
15815	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15816	break;
15817	}
15818
15819	/*
15820	* CLTS (Clear Task-Switch Flag in CR0).
15821	*/
15822	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15823	{
15824	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15825	break;
15826	}
15827
15828	/*
15829	* LMSW (Load Machine-Status Word into CR0).
15830	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15831	*/
15832	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15833	{
15834	RTGCPTR GCPtrEffDst;
15835	uint8_t const cbInstr = pVmxTransient->cbInstr;
15836	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15837	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15838	if (fMemOperand)
15839	{
15840	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
15841	AssertRCReturn(rc, rc);
15842	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15843	}
15844	else
15845	GCPtrEffDst = NIL_RTGCPTR;
15846	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15847	break;
15848	}
15849
15850	default:
15851	{
15852	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15853	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15854	}
15855	}
15856
15857	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15858	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15859	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15860
15861	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15862	NOREF(pVM);
15863	return rcStrict;
15864	}
15865
15866
15867	/**
15868	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15869	* VM-exit.
15870	*/
15871	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15872	{
15873	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15874	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15875
15876	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15877	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15878	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15879	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15880	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15881	\| CPUMCTX_EXTRN_EFER);
15882	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15883	AssertRCReturn(rc, rc);
15884
15885	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15886	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15887	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15888	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15889	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15890	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15891	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15892	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15893
15894	/*
15895	* Update exit history to see if this exit can be optimized.
15896	*/
15897	VBOXSTRICTRC rcStrict;
15898	PCEMEXITREC pExitRec = NULL;
15899	if ( !fGstStepping
15900	&& !fDbgStepping)
15901	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15902	!fIOString
15903	? !fIOWrite
15904	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15905	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15906	: !fIOWrite
15907	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15908	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15909	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15910	if (!pExitRec)
15911	{
15912	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15913	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15914
15915	uint32_t const cbValue = s_aIOSizes[uIOSize];
15916	uint32_t const cbInstr = pVmxTransient->cbInstr;
15917	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15918	PVM pVM = pVCpu->CTX_SUFF(pVM);
15919	if (fIOString)
15920	{
15921	/*
15922	* INS/OUTS - I/O String instruction.
15923	*
15924	* Use instruction-information if available, otherwise fall back on
15925	* interpreting the instruction.
15926	*/
15927	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15928	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15929	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15930	if (fInsOutsInfo)
15931	{
15932	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15933	AssertRCReturn(rc2, rc2);
15934	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15935	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15936	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15937	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15938	if (fIOWrite)
15939	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15940	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15941	else
15942	{
15943	/*
15944	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15945	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15946	* See Intel Instruction spec. for "INS".
15947	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15948	*/
15949	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15950	}
15951	}
15952	else
15953	rcStrict = IEMExecOne(pVCpu);
15954
15955	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15956	fUpdateRipAlready = true;
15957	}
15958	else
15959	{
15960	/*
15961	* IN/OUT - I/O instruction.
15962	*/
15963	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15964	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15965	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15966	if (fIOWrite)
15967	{
15968	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15969	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15970	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15971	&& !pCtx->eflags.Bits.u1TF)
15972	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15973	}
15974	else
15975	{
15976	uint32_t u32Result = 0;
15977	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15978	if (IOM_SUCCESS(rcStrict))
15979	{
15980	/* Save result of I/O IN instr. in AL/AX/EAX. */
15981	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15982	}
15983	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15984	&& !pCtx->eflags.Bits.u1TF)
15985	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15986	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15987	}
15988	}
15989
15990	if (IOM_SUCCESS(rcStrict))
15991	{
15992	if (!fUpdateRipAlready)
15993	{
15994	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15995	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15996	}
15997
15998	/*
15999	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
16000	* while booting Fedora 17 64-bit guest.
16001	*
16002	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
16003	*/
16004	if (fIOString)
16005	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
16006
16007	/*
16008	* If any I/O breakpoints are armed, we need to check if one triggered
16009	* and take appropriate action.
16010	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
16011	*/
16012	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
16013	AssertRCReturn(rc, rc);
16014
16015	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
16016	* execution engines about whether hyper BPs and such are pending. */
16017	uint32_t const uDr7 = pCtx->dr[7];
16018	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
16019	&& X86_DR7_ANY_RW_IO(uDr7)
16020	&& (pCtx->cr4 & X86_CR4_DE))
16021	\|\| DBGFBpIsHwIoArmed(pVM)))
16022	{
16023	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
16024
16025	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
16026	VMMRZCallRing3Disable(pVCpu);
16027	HM_DISABLE_PREEMPT(pVCpu);
16028
16029	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
16030
16031	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
16032	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
16033	{
16034	/* Raise #DB. */
16035	if (fIsGuestDbgActive)
16036	ASMSetDR6(pCtx->dr[6]);
16037	if (pCtx->dr[7] != uDr7)
16038	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
16039
16040	hmR0VmxSetPendingXcptDB(pVCpu);
16041	}
16042	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
16043	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
16044	else if ( rcStrict2 != VINF_SUCCESS
16045	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
16046	rcStrict = rcStrict2;
16047	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
16048
16049	HM_RESTORE_PREEMPT();
16050	VMMRZCallRing3Enable(pVCpu);
16051	}
16052	}
16053
16054	#ifdef VBOX_STRICT
16055	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
16056	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
16057	Assert(!fIOWrite);
16058	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
16059	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
16060	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
16061	Assert(fIOWrite);
16062	else
16063	{
16064	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
16065	* statuses, that the VMM device and some others may return. See
16066	* IOM_SUCCESS() for guidance. */
16067	AssertMsg( RT_FAILURE(rcStrict)
16068	\|\| rcStrict == VINF_SUCCESS
16069	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
16070	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
16071	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
16072	\|\| rcStrict == VINF_EM_RAW_TO_R3
16073	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16074	# endif
16075	}
16076	#endif
16077	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
16078	}
16079	else
16080	{
16081	/*
16082	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
16083	*/
16084	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16085	AssertRCReturn(rc2, rc2);
16086	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
16087	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
16088	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
16089	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16090	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
16091	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
16092
16093	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16094	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16095
16096	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16097	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16098	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16099	}
16100	return rcStrict;
16101	}
16102
16103
16104	/**
16105	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
16106	* VM-exit.
16107	*/
16108	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16109	{
16110	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16111
16112	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
16113	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16114	AssertRCReturn(rc, rc);
16115	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
16116	{
16117	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16118	AssertRCReturn(rc, rc);
16119	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16120	{
16121	uint32_t uErrCode;
16122	RTGCUINTPTR GCPtrFaultAddress;
16123	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
16124	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
16125	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
16126	if (fErrorCodeValid)
16127	{
16128	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16129	AssertRCReturn(rc, rc);
16130	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
16131	}
16132	else
16133	uErrCode = 0;
16134
16135	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
16136	&& uVector == X86_XCPT_PF)
16137	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
16138	else
16139	GCPtrFaultAddress = 0;
16140
16141	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16142	AssertRCReturn(rc, rc);
16143
16144	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
16145	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
16146
16147	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
16148	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16149	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16150	}
16151	}
16152
16153	/* Fall back to the interpreter to emulate the task-switch. */
16154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16155	return VERR_EM_INTERPRETER;
16156	}
16157
16158
16159	/**
16160	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
16161	*/
16162	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16163	{
16164	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16165
16166	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16167	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
16168	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16169	AssertRCReturn(rc, rc);
16170	return VINF_EM_DBG_STEPPED;
16171	}
16172
16173
16174	/**
16175	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
16176	*/
16177	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16178	{
16179	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16180	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
16181
16182	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16183	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16184	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16185	{
16186	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
16187	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16188	{
16189	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
16190	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16191	}
16192	}
16193	else
16194	{
16195	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16196	rcStrict1 = VINF_SUCCESS;
16197	return rcStrict1;
16198	}
16199
16200	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
16201	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16202	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16203	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16204	AssertRCReturn(rc, rc);
16205
16206	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
16207	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
16208	VBOXSTRICTRC rcStrict2;
16209	switch (uAccessType)
16210	{
16211	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
16212	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
16213	{
16214	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
16215	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
16216	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
16217
16218	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
16219	GCPhys &= PAGE_BASE_GC_MASK;
16220	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
16221	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
16222	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
16223
16224	PVM pVM = pVCpu->CTX_SUFF(pVM);
16225	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16226	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
16227	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
16228	CPUMCTX2CORE(pCtx), GCPhys);
16229	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
16230	if ( rcStrict2 == VINF_SUCCESS
16231	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
16232	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
16233	{
16234	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16235	\| HM_CHANGED_GUEST_APIC_TPR);
16236	rcStrict2 = VINF_SUCCESS;
16237	}
16238	break;
16239	}
16240
16241	default:
16242	{
16243	Log4Func(("uAccessType=%#x\n", uAccessType));
16244	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
16245	break;
16246	}
16247	}
16248
16249	if (rcStrict2 != VINF_SUCCESS)
16250	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
16251	return rcStrict2;
16252	}
16253
16254
16255	/**
16256	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
16257	* VM-exit.
16258	*/
16259	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16260	{
16261	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16262
16263	/* We should -not- get this VM-exit if the guest's debug registers were active. */
16264	if (pVmxTransient->fWasGuestDebugStateActive)
16265	{
16266	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
16267	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
16268	}
16269
16270	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16271	if ( !pVCpu->hm.s.fSingleInstruction
16272	&& !pVmxTransient->fWasHyperDebugStateActive)
16273	{
16274	Assert(!DBGFIsStepping(pVCpu));
16275	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
16276
16277	/* Don't intercept MOV DRx any more. */
16278	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
16279	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16280	AssertRCReturn(rc, rc);
16281
16282	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
16283	VMMRZCallRing3Disable(pVCpu);
16284	HM_DISABLE_PREEMPT(pVCpu);
16285
16286	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
16287	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
16288	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
16289
16290	HM_RESTORE_PREEMPT();
16291	VMMRZCallRing3Enable(pVCpu);
16292
16293	#ifdef VBOX_WITH_STATISTICS
16294	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16295	AssertRCReturn(rc, rc);
16296	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16297	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16298	else
16299	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16300	#endif
16301	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
16302	return VINF_SUCCESS;
16303	}
16304
16305	/*
16306	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
16307	* The EFER MSR is always up-to-date.
16308	* Update the segment registers and DR7 from the CPU.
16309	*/
16310	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16311	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16312	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
16313	AssertRCReturn(rc, rc);
16314	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
16315
16316	PVM pVM = pVCpu->CTX_SUFF(pVM);
16317	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16318	{
16319	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16320	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
16321	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
16322	if (RT_SUCCESS(rc))
16323	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
16324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16325	}
16326	else
16327	{
16328	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16329	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
16330	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
16331	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16332	}
16333
16334	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
16335	if (RT_SUCCESS(rc))
16336	{
16337	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
16338	AssertRCReturn(rc2, rc2);
16339	return VINF_SUCCESS;
16340	}
16341	return rc;
16342	}
16343
16344
16345	/**
16346	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
16347	* Conditional VM-exit.
16348	*/
16349	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16350	{
16351	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16352	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16353
16354	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16355	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16356	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16357	{
16358	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
16359	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
16360	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16361	{
16362	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
16363	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16364	}
16365	}
16366	else
16367	{
16368	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16369	rcStrict1 = VINF_SUCCESS;
16370	return rcStrict1;
16371	}
16372
16373	/*
16374	* Get sufficent state and update the exit history entry.
16375	*/
16376	RTGCPHYS GCPhys;
16377	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16378	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16379	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16380	AssertRCReturn(rc, rc);
16381
16382	VBOXSTRICTRC rcStrict;
16383	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
16384	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
16385	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
16386	if (!pExitRec)
16387	{
16388	/*
16389	* If we succeed, resume guest execution.
16390	* If we fail in interpreting the instruction because we couldn't get the guest physical address
16391	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
16392	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
16393	* weird case. See @bugref{6043}.
16394	*/
16395	PVM pVM = pVCpu->CTX_SUFF(pVM);
16396	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16397	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
16398	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
16399	if ( rcStrict == VINF_SUCCESS
16400	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16401	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16402	{
16403	/* Successfully handled MMIO operation. */
16404	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16405	\| HM_CHANGED_GUEST_APIC_TPR);
16406	rcStrict = VINF_SUCCESS;
16407	}
16408	}
16409	else
16410	{
16411	/*
16412	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
16413	*/
16414	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16415	AssertRCReturn(rc2, rc2);
16416
16417	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
16418	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
16419
16420	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16421	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16422
16423	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16424	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16425	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16426	}
16427	return rcStrict;
16428	}
16429
16430
16431	/**
16432	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
16433	* VM-exit.
16434	*/
16435	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16436	{
16437	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16438	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16439
16440	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16441	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16442	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16443	{
16444	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
16445	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16446	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
16447	}
16448	else
16449	{
16450	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16451	rcStrict1 = VINF_SUCCESS;
16452	return rcStrict1;
16453	}
16454
16455	RTGCPHYS GCPhys;
16456	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16457	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16458	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16459	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16460	AssertRCReturn(rc, rc);
16461
16462	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
16463	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
16464
16465	RTGCUINT uErrorCode = 0;
16466	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16467	uErrorCode \|= X86_TRAP_PF_ID;
16468	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16469	uErrorCode \|= X86_TRAP_PF_RW;
16470	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16471	uErrorCode \|= X86_TRAP_PF_P;
16472
16473	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16474
16475
16476	/* Handle the pagefault trap for the nested shadow table. */
16477	PVM pVM = pVCpu->CTX_SUFF(pVM);
16478	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16479
16480	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
16481	pCtx->cs.Sel, pCtx->rip));
16482
16483	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16484	TRPMResetTrap(pVCpu);
16485
16486	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16487	if ( rcStrict2 == VINF_SUCCESS
16488	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
16489	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
16490	{
16491	/* Successfully synced our nested page tables. */
16492	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16494	return VINF_SUCCESS;
16495	}
16496
16497	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
16498	return rcStrict2;
16499	}
16500
16501
16502	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16503	/**
16504	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16505	*/
16506	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16507	{
16508	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16509
16510	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16511	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16512	\| CPUMCTX_EXTRN_HWVIRT
16513	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16514	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16515	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16516	AssertRCReturn(rc, rc);
16517
16518	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16519
16520	VMXVEXITINFO ExitInfo;
16521	RT_ZERO(ExitInfo);
16522	ExitInfo.uReason = pVmxTransient->uExitReason;
16523	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16524	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16525	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16526	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16527
16528	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16529	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16530	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16531	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16532	{
16533	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16534	rcStrict = VINF_SUCCESS;
16535	}
16536	return rcStrict;
16537	}
16538
16539
16540	/**
16541	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16542	*/
16543	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16544	{
16545	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16546
16547	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16548	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16549	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16550	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16551	AssertRCReturn(rc, rc);
16552
16553	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16554
16555	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16556	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16557	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16558	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16559	{
16560	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16561	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16562	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16563	}
16564	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16565	return rcStrict;
16566	}
16567
16568
16569	/**
16570	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16571	*/
16572	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16573	{
16574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16575
16576	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16577	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16578	\| CPUMCTX_EXTRN_HWVIRT
16579	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16580	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16581	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16582	AssertRCReturn(rc, rc);
16583
16584	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16585
16586	VMXVEXITINFO ExitInfo;
16587	RT_ZERO(ExitInfo);
16588	ExitInfo.uReason = pVmxTransient->uExitReason;
16589	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16590	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16591	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16592	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16593
16594	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16595	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16596	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16597	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16598	{
16599	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16600	rcStrict = VINF_SUCCESS;
16601	}
16602	return rcStrict;
16603	}
16604
16605
16606	/**
16607	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16608	*/
16609	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16610	{
16611	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16612
16613	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16614	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16615	\| CPUMCTX_EXTRN_HWVIRT
16616	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16617	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16618	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16619	AssertRCReturn(rc, rc);
16620
16621	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16622
16623	VMXVEXITINFO ExitInfo;
16624	RT_ZERO(ExitInfo);
16625	ExitInfo.uReason = pVmxTransient->uExitReason;
16626	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16627	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16628	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16629	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16630
16631	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16632	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16633	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16634	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16635	{
16636	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16637	rcStrict = VINF_SUCCESS;
16638	}
16639	return rcStrict;
16640	}
16641
16642
16643	/**
16644	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16645	*/
16646	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16647	{
16648	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16649
16650	/*
16651	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16652	* thus might not need to import the shadow VMCS state, it's safer just in case
16653	* code elsewhere dares look at unsynced VMCS fields.
16654	*/
16655	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16656	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16657	\| CPUMCTX_EXTRN_HWVIRT
16658	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16659	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16660	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16661	AssertRCReturn(rc, rc);
16662
16663	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16664
16665	VMXVEXITINFO ExitInfo;
16666	RT_ZERO(ExitInfo);
16667	ExitInfo.uReason = pVmxTransient->uExitReason;
16668	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16669	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16670	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16671	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16672	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16673
16674	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16675	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16676	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16677	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16678	{
16679	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16680	rcStrict = VINF_SUCCESS;
16681	}
16682	return rcStrict;
16683	}
16684
16685
16686	/**
16687	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16688	*/
16689	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16690	{
16691	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16692
16693	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16694	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16695	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16696	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16697	AssertRCReturn(rc, rc);
16698
16699	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16700
16701	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16702	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16703	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16704	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16705	{
16706	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16707	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16708	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16709	}
16710	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16711	return rcStrict;
16712	}
16713
16714
16715	/**
16716	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16717	*/
16718	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16719	{
16720	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16721
16722	/*
16723	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16724	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16725	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16726	*/
16727	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16728	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16729	\| CPUMCTX_EXTRN_HWVIRT
16730	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16731	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16732	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16733	AssertRCReturn(rc, rc);
16734
16735	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16736
16737	VMXVEXITINFO ExitInfo;
16738	RT_ZERO(ExitInfo);
16739	ExitInfo.uReason = pVmxTransient->uExitReason;
16740	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16741	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16742	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16743	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16744	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16745
16746	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16747	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16748	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16749	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16750	{
16751	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16752	rcStrict = VINF_SUCCESS;
16753	}
16754	return rcStrict;
16755	}
16756
16757
16758	/**
16759	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16760	*/
16761	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16762	{
16763	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16764
16765	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16766	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16767	\| CPUMCTX_EXTRN_HWVIRT
16768	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16769	AssertRCReturn(rc, rc);
16770
16771	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16772
16773	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16774	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16775	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16776	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16777	{
16778	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16779	rcStrict = VINF_SUCCESS;
16780	}
16781	return rcStrict;
16782	}
16783
16784
16785	/**
16786	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16787	*/
16788	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16789	{
16790	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16791
16792	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16793	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16794	\| CPUMCTX_EXTRN_HWVIRT
16795	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16796	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16797	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16798	AssertRCReturn(rc, rc);
16799
16800	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16801
16802	VMXVEXITINFO ExitInfo;
16803	RT_ZERO(ExitInfo);
16804	ExitInfo.uReason = pVmxTransient->uExitReason;
16805	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16806	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16807	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16808	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16809
16810	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16811	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16812	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16813	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16814	{
16815	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16816	rcStrict = VINF_SUCCESS;
16817	}
16818	return rcStrict;
16819	}
16820
16821
16822	/**
16823	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16824	*/
16825	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16826	{
16827	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16828
16829	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16830	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16831	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16832	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16833	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16834	AssertRCReturn(rc, rc);
16835
16836	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16837
16838	VMXVEXITINFO ExitInfo;
16839	RT_ZERO(ExitInfo);
16840	ExitInfo.uReason = pVmxTransient->uExitReason;
16841	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16842	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16843	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16844	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16845
16846	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16847	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16848	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16849	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16850	{
16851	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16852	rcStrict = VINF_SUCCESS;
16853	}
16854	return rcStrict;
16855	}
16856	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16857	/** @} */
16858
16859
16860	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16861	/** @name Nested-guest VM-exit handlers.
16862	* @{
16863	*/
16864	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16865	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16866	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16867
16868	/**
16869	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16870	* Conditional VM-exit.
16871	*/
16872	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16873	{
16874	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16875
16876	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16877	AssertRCReturn(rc, rc);
16878
16879	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16880	uint32_t const uExtIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16881	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16882
16883	/*
16884	* Make sure not to use stale/previous VM-exit instruction length since we read the
16885	* instruction length from the VMCS below only for software exceptions and privileged
16886	* software exceptions but we pass it for all exception VM-exits below.
16887	*/
16888	pVmxTransient->cbInstr = 0;
16889
16890	switch (uExtIntType)
16891	{
16892	/*
16893	* Physical NMIs:
16894	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16895	*/
16896	case VMX_EXIT_INT_INFO_TYPE_NMI:
16897	return hmR0VmxExitHostNmi(pVCpu);
16898
16899	/*
16900	* Hardware exceptions,
16901	* Software exceptions,
16902	* Privileged software exceptions:
16903	* Figure out if the exception must be delivered to the guest or the nested-guest.
16904	*
16905	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16906	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16907	* length.
16908	*/
16909	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16910	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16911	{
16912	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16913	AssertRCReturn(rc, rc);
16914	RT_FALL_THRU();
16915	}
16916	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16917	{
16918	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16919	AssertRCReturn(rc, rc);
16920
16921	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
16922	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
16923	pVmxTransient->uExitIntErrorCode);
16924	if (fIntercept)
16925	{
16926	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16927	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16928	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16929	AssertRCReturn(rc, rc);
16930
16931	VMXVEXITINFO ExitInfo;
16932	RT_ZERO(ExitInfo);
16933	ExitInfo.uReason = pVmxTransient->uExitReason;
16934	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16935	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16936
16937	VMXVEXITEVENTINFO ExitEventInfo;
16938	RT_ZERO(ExitEventInfo);
16939	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16940	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16941	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16942	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16943
16944	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16945	}
16946
16947	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
16948	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16949
16950	/* If the guest hypervisor is not intercepting the exception, forward it to the guest. */
16951	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
16952	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
16953	return VINF_SUCCESS;
16954	}
16955
16956	/*
16957	* Software interrupts:
16958	* VM-exits cannot be caused by software interrupts.
16959	*
16960	* External interrupts:
16961	* This should only happen when "acknowledge external interrupts on VM-exit"
16962	* control is set. However, we never set this when executing a guest or
16963	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16964	* the guest.
16965	*/
16966	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16967	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16968	default:
16969	{
16970	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16971	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16972	}
16973	}
16974	}
16975
16976
16977	/**
16978	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16979	* Unconditional VM-exit.
16980	*/
16981	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16982	{
16983	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16984	return IEMExecVmxVmexitTripleFault(pVCpu);
16985	}
16986
16987
16988	/**
16989	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16990	*/
16991	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16992	{
16993	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16994
16995	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16996	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16997	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16998	}
16999
17000
17001	/**
17002	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
17003	*/
17004	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17005	{
17006	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17007
17008	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
17009	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17010	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
17011	}
17012
17013
17014	/**
17015	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
17016	* Unconditional VM-exit.
17017	*/
17018	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17019	{
17020	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17021
17022	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17023	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17024	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17025	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17026	AssertRCReturn(rc, rc);
17027
17028	VMXVEXITINFO ExitInfo;
17029	RT_ZERO(ExitInfo);
17030	ExitInfo.uReason = pVmxTransient->uExitReason;
17031	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17032	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17033
17034	VMXVEXITEVENTINFO ExitEventInfo;
17035	RT_ZERO(ExitEventInfo);
17036	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17037	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17038	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
17039	}
17040
17041
17042	/**
17043	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
17044	*/
17045	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17046	{
17047	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17048
17049	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
17050	{
17051	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17052	AssertRCReturn(rc, rc);
17053	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17054	}
17055	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
17056	}
17057
17058
17059	/**
17060	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
17061	*/
17062	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17063	{
17064	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17065
17066	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17067	{
17068	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17069	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17070	AssertRCReturn(rc, rc);
17071
17072	VMXVEXITINFO ExitInfo;
17073	RT_ZERO(ExitInfo);
17074	ExitInfo.uReason = pVmxTransient->uExitReason;
17075	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17076	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17077	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17078	}
17079	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
17080	}
17081
17082
17083	/**
17084	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
17085	*/
17086	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17087	{
17088	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17089
17090	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
17091	{
17092	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17093	AssertRCReturn(rc, rc);
17094	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17095	}
17096	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
17097	}
17098
17099
17100	/**
17101	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
17102	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
17103	*/
17104	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17105	{
17106	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17107
17108	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
17109	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
17110
17111	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17112	AssertRCReturn(rc, rc);
17113
17114	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
17115	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17116	uint64_t u64FieldEnc = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17117
17118	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
17119	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
17120	u64FieldEnc &= UINT64_C(0xffffffff);
17121
17122	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64FieldEnc))
17123	{
17124	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17125	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17126	AssertRCReturn(rc, rc);
17127
17128	VMXVEXITINFO ExitInfo;
17129	RT_ZERO(ExitInfo);
17130	ExitInfo.uReason = pVmxTransient->uExitReason;
17131	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17132	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17133	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17134	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17135	}
17136
17137	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
17138	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
17139	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
17140	}
17141
17142
17143	/**
17144	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
17145	*/
17146	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17147	{
17148	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17149
17150	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17151	{
17152	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17153	AssertRCReturn(rc, rc);
17154	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17155	}
17156
17157	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
17158	}
17159
17160
17161	/**
17162	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
17163	* Conditional VM-exit.
17164	*/
17165	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17166	{
17167	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17168
17169	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17170	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17171	AssertRCReturn(rc, rc);
17172
17173	VBOXSTRICTRC rcStrict;
17174	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
17175	switch (uAccessType)
17176	{
17177	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
17178	{
17179	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17180	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17181	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17182	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17183
17184	bool fIntercept;
17185	switch (iCrReg)
17186	{
17187	case 0:
17188	case 4:
17189	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
17190	break;
17191
17192	case 3:
17193	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
17194	break;
17195
17196	case 8:
17197	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
17198	break;
17199
17200	default:
17201	fIntercept = false;
17202	break;
17203	}
17204	if (fIntercept)
17205	{
17206	VMXVEXITINFO ExitInfo;
17207	RT_ZERO(ExitInfo);
17208	ExitInfo.uReason = pVmxTransient->uExitReason;
17209	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17210	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17211	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17212	}
17213	else
17214	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17215	break;
17216	}
17217
17218	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
17219	{
17220	/*
17221	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
17222	* CR2 reads do not cause a VM-exit.
17223	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
17224	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
17225	*/
17226	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17227	if ( iCrReg == 3
17228	\|\| iCrReg == 8)
17229	{
17230	static const uint32_t s_aCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
17231	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
17232	uint32_t const uIntercept = s_aCrXReadIntercepts[iCrReg];
17233	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
17234	{
17235	VMXVEXITINFO ExitInfo;
17236	RT_ZERO(ExitInfo);
17237	ExitInfo.uReason = pVmxTransient->uExitReason;
17238	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17239	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17240	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17241	}
17242	else
17243	{
17244	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17245	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17246	}
17247	}
17248	else
17249	{
17250	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
17251	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
17252	}
17253	break;
17254	}
17255
17256	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
17257	{
17258	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
17259	Assert(pVmcsNstGst);
17260	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
17261	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
17262	if ( (uGstHostMask & X86_CR0_TS)
17263	&& (uReadShadow & X86_CR0_TS))
17264	{
17265	VMXVEXITINFO ExitInfo;
17266	RT_ZERO(ExitInfo);
17267	ExitInfo.uReason = pVmxTransient->uExitReason;
17268	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17269	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17270	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17271	}
17272	else
17273	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
17274	break;
17275	}
17276
17277	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
17278	{
17279	RTGCPTR GCPtrEffDst;
17280	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
17281	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
17282	if (fMemOperand)
17283	{
17284	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17285	AssertRCReturn(rc, rc);
17286	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
17287	}
17288	else
17289	GCPtrEffDst = NIL_RTGCPTR;
17290
17291	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
17292	{
17293	VMXVEXITINFO ExitInfo;
17294	RT_ZERO(ExitInfo);
17295	ExitInfo.uReason = pVmxTransient->uExitReason;
17296	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17297	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
17298	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17299	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17300	}
17301	else
17302	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
17303	break;
17304	}
17305
17306	default:
17307	{
17308	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
17309	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
17310	}
17311	}
17312
17313	if (rcStrict == VINF_IEM_RAISED_XCPT)
17314	{
17315	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
17316	rcStrict = VINF_SUCCESS;
17317	}
17318	return rcStrict;
17319	}
17320
17321
17322	/**
17323	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
17324	* Conditional VM-exit.
17325	*/
17326	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17327	{
17328	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17329
17330	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
17331	{
17332	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17333	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17334	AssertRCReturn(rc, rc);
17335
17336	VMXVEXITINFO ExitInfo;
17337	RT_ZERO(ExitInfo);
17338	ExitInfo.uReason = pVmxTransient->uExitReason;
17339	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17340	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17341	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17342	}
17343	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
17344	}
17345
17346
17347	/**
17348	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
17349	* Conditional VM-exit.
17350	*/
17351	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17352	{
17353	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17354
17355	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17356	AssertRCReturn(rc, rc);
17357
17358	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
17359	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
17360	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
17361
17362	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
17363	uint8_t const cbAccess = s_aIOSizes[uIOSize];
17364	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
17365	{
17366	/*
17367	* IN/OUT instruction:
17368	* - Provides VM-exit instruction length.
17369	*
17370	* INS/OUTS instruction:
17371	* - Provides VM-exit instruction length.
17372	* - Provides Guest-linear address.
17373	* - Optionally provides VM-exit instruction info (depends on CPU feature).
17374	*/
17375	PVM pVM = pVCpu->CTX_SUFF(pVM);
17376	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17377	AssertRCReturn(rc, rc);
17378
17379	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
17380	pVmxTransient->ExitInstrInfo.u = 0;
17381	pVmxTransient->uGuestLinearAddr = 0;
17382
17383	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
17384	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
17385	if (fIOString)
17386	{
17387	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17388	if (fVmxInsOutsInfo)
17389	{
17390	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
17391	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17392	}
17393	}
17394	AssertRCReturn(rc, rc);
17395
17396	VMXVEXITINFO ExitInfo;
17397	RT_ZERO(ExitInfo);
17398	ExitInfo.uReason = pVmxTransient->uExitReason;
17399	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17400	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17401	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17402	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
17403	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17404	}
17405	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
17406	}
17407
17408
17409	/**
17410	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
17411	*/
17412	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17413	{
17414	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17415
17416	uint32_t fMsrpm;
17417	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17418	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17419	else
17420	fMsrpm = VMXMSRPM_EXIT_RD;
17421
17422	if (fMsrpm & VMXMSRPM_EXIT_RD)
17423	{
17424	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17425	AssertRCReturn(rc, rc);
17426	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17427	}
17428	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
17429	}
17430
17431
17432	/**
17433	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
17434	*/
17435	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17436	{
17437	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17438
17439	uint32_t fMsrpm;
17440	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17441	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17442	else
17443	fMsrpm = VMXMSRPM_EXIT_WR;
17444
17445	if (fMsrpm & VMXMSRPM_EXIT_WR)
17446	{
17447	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17448	AssertRCReturn(rc, rc);
17449	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17450	}
17451	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17452	}
17453
17454
17455	/**
17456	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17457	*/
17458	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17459	{
17460	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17461
17462	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17463	{
17464	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17465	AssertRCReturn(rc, rc);
17466	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17467	}
17468	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17469	}
17470
17471
17472	/**
17473	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17474	* VM-exit.
17475	*/
17476	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17477	{
17478	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17479
17480	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17481	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17482	}
17483
17484
17485	/**
17486	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17487	*/
17488	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17489	{
17490	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17491
17492	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17493	{
17494	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17495	AssertRCReturn(rc, rc);
17496	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17497	}
17498	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17499	}
17500
17501
17502	/**
17503	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17504	*/
17505	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17506	{
17507	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17508
17509	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17510	* PAUSE when executing a nested-guest? If it does not, we would not need
17511	* to check for the intercepts here. Just call VM-exit... */
17512
17513	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17514	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17515	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17516	{
17517	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17518	AssertRCReturn(rc, rc);
17519	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17520	}
17521	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17522	}
17523
17524
17525	/**
17526	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17527	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17528	*/
17529	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17530	{
17531	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17532
17533	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17534	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17535	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17536	}
17537
17538
17539	/**
17540	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17541	* VM-exit.
17542	*/
17543	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17544	{
17545	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17546
17547	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17548	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17549	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17550	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17551	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17552	AssertRCReturn(rc, rc);
17553
17554	VMXVEXITINFO ExitInfo;
17555	RT_ZERO(ExitInfo);
17556	ExitInfo.uReason = pVmxTransient->uExitReason;
17557	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17558	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17559
17560	VMXVEXITEVENTINFO ExitEventInfo;
17561	RT_ZERO(ExitEventInfo);
17562	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17563	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17564	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17565	}
17566
17567
17568	/**
17569	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17570	* Conditional VM-exit.
17571	*/
17572	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17573	{
17574	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17575
17576	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17577	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17578	AssertRCReturn(rc, rc);
17579
17580	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17581	}
17582
17583
17584	/**
17585	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17586	* Conditional VM-exit.
17587	*/
17588	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17589	{
17590	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17591
17592	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17593	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17594	AssertRCReturn(rc, rc);
17595
17596	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17597	}
17598
17599
17600	/**
17601	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17602	*/
17603	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17604	{
17605	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17606
17607	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17608	{
17609	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17610	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17611	AssertRCReturn(rc, rc);
17612	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17613	}
17614	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17615	}
17616
17617
17618	/**
17619	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17620	*/
17621	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17622	{
17623	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17624
17625	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17626	{
17627	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17628	AssertRCReturn(rc, rc);
17629	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17630	}
17631	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17632	}
17633
17634
17635	/**
17636	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17637	*/
17638	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17639	{
17640	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17641
17642	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17643	{
17644	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17645	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17646	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17647	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17648	AssertRCReturn(rc, rc);
17649
17650	VMXVEXITINFO ExitInfo;
17651	RT_ZERO(ExitInfo);
17652	ExitInfo.uReason = pVmxTransient->uExitReason;
17653	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17654	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17655	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17656	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17657	}
17658	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17659	}
17660
17661
17662	/**
17663	* Nested-guest VM-exit handler for invalid-guest state
17664	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17665	*/
17666	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17667	{
17668	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17669
17670	/*
17671	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17672	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17673	* Handle it like it's in an invalid guest state of the outer guest.
17674	*
17675	* When the fast path is implemented, this should be changed to cause the corresponding
17676	* nested-guest VM-exit.
17677	*/
17678	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17679	}
17680
17681
17682	/**
17683	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17684	* and only provide the instruction length.
17685	*
17686	* Unconditional VM-exit.
17687	*/
17688	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17689	{
17690	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17691
17692	#ifdef VBOX_STRICT
17693	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17694	switch (pVmxTransient->uExitReason)
17695	{
17696	case VMX_EXIT_ENCLS:
17697	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17698	break;
17699
17700	case VMX_EXIT_VMFUNC:
17701	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17702	break;
17703	}
17704	#endif
17705
17706	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17707	AssertRCReturn(rc, rc);
17708	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17709	}
17710
17711
17712	/**
17713	* Nested-guest VM-exit handler for instructions that provide instruction length as
17714	* well as more information.
17715	*
17716	* Unconditional VM-exit.
17717	*/
17718	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17719	{
17720	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17721
17722	#ifdef VBOX_STRICT
17723	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17724	switch (pVmxTransient->uExitReason)
17725	{
17726	case VMX_EXIT_GDTR_IDTR_ACCESS:
17727	case VMX_EXIT_LDTR_TR_ACCESS:
17728	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17729	break;
17730
17731	case VMX_EXIT_RDRAND:
17732	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17733	break;
17734
17735	case VMX_EXIT_RDSEED:
17736	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17737	break;
17738
17739	case VMX_EXIT_XSAVES:
17740	case VMX_EXIT_XRSTORS:
17741	/** @todo NSTVMX: Verify XSS-bitmap. */
17742	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17743	break;
17744
17745	case VMX_EXIT_UMWAIT:
17746	case VMX_EXIT_TPAUSE:
17747	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17748	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17749	break;
17750	}
17751	#endif
17752
17753	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17754	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17755	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17756	AssertRCReturn(rc, rc);
17757
17758	VMXVEXITINFO ExitInfo;
17759	RT_ZERO(ExitInfo);
17760	ExitInfo.uReason = pVmxTransient->uExitReason;
17761	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17762	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17763	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17764	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17765	}
17766
17767	/** @} */
17768	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17769

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79564

Download in other formats: