HMVMXR0.cpp@ 72390

Last change on this file since 72390 was 72390, checked in by vboxsync, 7 years ago
VMM/HMVMXR0: Workaround erratum with INVVPID outside 64-bit mode not invalidating 64-bit linear addresses.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 601.0 KB

Line
1	/* $Id: HMVMXR0.cpp 72390 2018-05-30 10:33:09Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#define HMVMX_USE_IEM_EVENT_REFLECTION
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50	# define HMVMX_ALWAYS_TRAP_PF
51	# define HMVMX_ALWAYS_SWAP_FPU_STATE
52	# define HMVMX_ALWAYS_FLUSH_TLB
53	# define HMVMX_ALWAYS_SWAP_EFER
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DR7 \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_INTR_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Exception bitmap mask for real-mode guests (real-on-v86).
140	*
141	* We need to intercept all exceptions manually except:
142	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144	* due to bugs in Intel CPUs.
145	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
146	* support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*********************************************************************************************************************************
197	* Structures and Typedefs *
198	*********************************************************************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG fEFlags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit code qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u7Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	/** INVEPT, INVVPID, INVPCID information. */
247	struct
248	{
249	/** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
250	uint32_t u2Scaling : 2;
251	uint32_t u5Reserved0 : 5;
252	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253	uint32_t u3AddrSize : 3;
254	uint32_t u1Reserved0 : 1;
255	uint32_t u4Reserved0 : 4;
256	/** The segment register (X86_SREG_XXX). */
257	uint32_t iSegReg : 3;
258	/** The index register (X86_GREG_XXX). */
259	uint32_t iIdxReg : 4;
260	/** Set if index register is invalid. */
261	uint32_t fIdxRegValid : 1;
262	/** The base register (X86_GREG_XXX). */
263	uint32_t iBaseReg : 4;
264	/** Set if base register is invalid. */
265	uint32_t fBaseRegValid : 1;
266	/** Register 2 (X86_GREG_XXX). */
267	uint32_t iReg2 : 4;
268	} Inv;
269	} ExitInstrInfo;
270	/** Whether the VM-entry failed or not. */
271	bool fVMEntryFailed;
272	/** Alignment. */
273	uint8_t abAlignment1[3];
274
275	/** The VM-entry interruption-information field. */
276	uint32_t uEntryIntInfo;
277	/** The VM-entry exception error code field. */
278	uint32_t uEntryXcptErrorCode;
279	/** The VM-entry instruction length field. */
280	uint32_t cbEntryInstr;
281
282	/** IDT-vectoring information field. */
283	uint32_t uIdtVectoringInfo;
284	/** IDT-vectoring error code. */
285	uint32_t uIdtVectoringErrorCode;
286
287	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
288	uint32_t fVmcsFieldsRead;
289
290	/** Whether the guest FPU was active at the time of VM-exit. */
291	bool fWasGuestFPUStateActive;
292	/** Whether the guest debug state was active at the time of VM-exit. */
293	bool fWasGuestDebugStateActive;
294	/** Whether the hyper debug state was active at the time of VM-exit. */
295	bool fWasHyperDebugStateActive;
296	/** Whether TSC-offsetting should be setup before VM-entry. */
297	bool fUpdateTscOffsettingAndPreemptTimer;
298	/** Whether the VM-exit was caused by a page-fault during delivery of a
299	* contributory exception or a page-fault. */
300	bool fVectoringDoublePF;
301	/** Whether the VM-exit was caused by a page-fault during delivery of an
302	* external interrupt or NMI. */
303	bool fVectoringPF;
304	} VMXTRANSIENT;
305	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
306	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
307	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
308	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
309	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
310	/** Pointer to VMX transient state. */
311	typedef VMXTRANSIENT *PVMXTRANSIENT;
312
313
314	/**
315	* MSR-bitmap read permissions.
316	*/
317	typedef enum VMXMSREXITREAD
318	{
319	/** Reading this MSR causes a VM-exit. */
320	VMXMSREXIT_INTERCEPT_READ = 0xb,
321	/** Reading this MSR does not cause a VM-exit. */
322	VMXMSREXIT_PASSTHRU_READ
323	} VMXMSREXITREAD;
324	/** Pointer to MSR-bitmap read permissions. */
325	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
326
327	/**
328	* MSR-bitmap write permissions.
329	*/
330	typedef enum VMXMSREXITWRITE
331	{
332	/** Writing to this MSR causes a VM-exit. */
333	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
334	/** Writing to this MSR does not cause a VM-exit. */
335	VMXMSREXIT_PASSTHRU_WRITE
336	} VMXMSREXITWRITE;
337	/** Pointer to MSR-bitmap write permissions. */
338	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
339
340
341	/**
342	* VMX VM-exit handler.
343	*
344	* @returns Strict VBox status code (i.e. informational status codes too).
345	* @param pVCpu The cross context virtual CPU structure.
346	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
347	* out-of-sync. Make sure to update the required
348	* fields before using them.
349	* @param pVmxTransient Pointer to the VMX-transient structure.
350	*/
351	#ifndef HMVMX_USE_FUNCTION_TABLE
352	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353	#else
354	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355	/** Pointer to VM-exit handler. */
356	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
357	#endif
358
359	/**
360	* VMX VM-exit handler, non-strict status code.
361	*
362	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
363	*
364	* @returns VBox status code, no informational status code returned.
365	* @param pVCpu The cross context virtual CPU structure.
366	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
367	* out-of-sync. Make sure to update the required
368	* fields before using them.
369	* @param pVmxTransient Pointer to the VMX-transient structure.
370	*
371	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
372	* use of that status code will be replaced with VINF_EM_SOMETHING
373	* later when switching over to IEM.
374	*/
375	#ifndef HMVMX_USE_FUNCTION_TABLE
376	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
377	#else
378	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
379	#endif
380
381
382	/*********************************************************************************************************************************
383	* Internal Functions *
384	*********************************************************************************************************************************/
385	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
386	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
387	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
388	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
389	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
390	bool fStepping, uint32_t *puIntState);
391	#if HC_ARCH_BITS == 32
392	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
393	#endif
394	#ifndef HMVMX_USE_FUNCTION_TABLE
395	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
396	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
397	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
398	#else
399	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
400	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
401	#endif
402
403
404	/** @name VM-exit handlers.
405	* @{
406	*/
407	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
408	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
409	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
415	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
416	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
417	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
418	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
419	static FNVMXEXITHANDLER hmR0VmxExitHlt;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
421	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
422	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
423	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
424	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
425	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
426	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
427	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
428	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
429	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
430	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
431	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
432	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
433	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
434	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
435	static FNVMXEXITHANDLER hmR0VmxExitMwait;
436	static FNVMXEXITHANDLER hmR0VmxExitMtf;
437	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
438	static FNVMXEXITHANDLER hmR0VmxExitPause;
439	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
440	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
441	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
442	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
443	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
444	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
445	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
446	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
447	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
448	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
449	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
450	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
451	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
452	/** @} */
453
454	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
458	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
459	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
460	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
461	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
462	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
463
464
465	/*********************************************************************************************************************************
466	* Global Variables *
467	*********************************************************************************************************************************/
468	#ifdef HMVMX_USE_FUNCTION_TABLE
469
470	/**
471	* VMX_EXIT dispatch table.
472	*/
473	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
474	{
475	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
476	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
477	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
478	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
479	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
480	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
481	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
482	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
483	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
484	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
485	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
486	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
487	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
488	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
489	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
490	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
491	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
492	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
493	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
494	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
495	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
496	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
497	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
498	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
499	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
500	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
501	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
502	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
503	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
504	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
505	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
506	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
507	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
508	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
509	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
510	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
511	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
512	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
513	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
514	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
515	/* 40 UNDEFINED */ hmR0VmxExitPause,
516	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
517	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
518	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
519	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
520	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
521	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
522	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
523	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
524	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
525	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
526	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
527	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
528	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
529	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
530	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
531	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
532	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
533	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
534	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
535	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
536	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
537	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
538	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
539	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
540	};
541	#endif /* HMVMX_USE_FUNCTION_TABLE */
542
543	#ifdef VBOX_STRICT
544	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
545	{
546	/* 0 */ "(Not Used)",
547	/* 1 */ "VMCALL executed in VMX root operation.",
548	/* 2 */ "VMCLEAR with invalid physical address.",
549	/* 3 */ "VMCLEAR with VMXON pointer.",
550	/* 4 */ "VMLAUNCH with non-clear VMCS.",
551	/* 5 */ "VMRESUME with non-launched VMCS.",
552	/* 6 */ "VMRESUME after VMXOFF",
553	/* 7 */ "VM-entry with invalid control fields.",
554	/* 8 */ "VM-entry with invalid host state fields.",
555	/* 9 */ "VMPTRLD with invalid physical address.",
556	/* 10 */ "VMPTRLD with VMXON pointer.",
557	/* 11 */ "VMPTRLD with incorrect revision identifier.",
558	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
559	/* 13 */ "VMWRITE to read-only VMCS component.",
560	/* 14 */ "(Not Used)",
561	/* 15 */ "VMXON executed in VMX root operation.",
562	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
563	/* 17 */ "VM-entry with non-launched executing VMCS.",
564	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
565	/* 19 */ "VMCALL with non-clear VMCS.",
566	/* 20 */ "VMCALL with invalid VM-exit control fields.",
567	/* 21 */ "(Not Used)",
568	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
569	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
570	/* 24 */ "VMCALL with invalid SMM-monitor features.",
571	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
572	/* 26 */ "VM-entry with events blocked by MOV SS.",
573	/* 27 */ "(Not Used)",
574	/* 28 */ "Invalid operand to INVEPT/INVVPID."
575	};
576	#endif /* VBOX_STRICT */
577
578
579
580	/**
581	* Updates the VM's last error record.
582	*
583	* If there was a VMX instruction error, reads the error data from the VMCS and
584	* updates VCPU's last error record as well.
585	*
586	* @param pVM The cross context VM structure.
587	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
588	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
589	* VERR_VMX_INVALID_VMCS_FIELD.
590	* @param rc The error code.
591	*/
592	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
593	{
594	AssertPtr(pVM);
595	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
596	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
597	{
598	AssertPtrReturnVoid(pVCpu);
599	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
600	}
601	pVM->hm.s.lLastError = rc;
602	}
603
604
605	/**
606	* Reads the VM-entry interruption-information field from the VMCS into the VMX
607	* transient structure.
608	*
609	* @returns VBox status code.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*
612	* @remarks No-long-jump zone!!!
613	*/
614	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
615	{
616	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
617	AssertRCReturn(rc, rc);
618	return VINF_SUCCESS;
619	}
620
621
622	#ifdef VBOX_STRICT
623	/**
624	* Reads the VM-entry exception error code field from the VMCS into
625	* the VMX transient structure.
626	*
627	* @returns VBox status code.
628	* @param pVmxTransient Pointer to the VMX transient structure.
629	*
630	* @remarks No-long-jump zone!!!
631	*/
632	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
633	{
634	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
635	AssertRCReturn(rc, rc);
636	return VINF_SUCCESS;
637	}
638	#endif /* VBOX_STRICT */
639
640
641	#ifdef VBOX_STRICT
642	/**
643	* Reads the VM-entry exception error code field from the VMCS into
644	* the VMX transient structure.
645	*
646	* @returns VBox status code.
647	* @param pVmxTransient Pointer to the VMX transient structure.
648	*
649	* @remarks No-long-jump zone!!!
650	*/
651	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
652	{
653	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
654	AssertRCReturn(rc, rc);
655	return VINF_SUCCESS;
656	}
657	#endif /* VBOX_STRICT */
658
659
660	/**
661	* Reads the VM-exit interruption-information field from the VMCS into the VMX
662	* transient structure.
663	*
664	* @returns VBox status code.
665	* @param pVmxTransient Pointer to the VMX transient structure.
666	*/
667	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
668	{
669	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
670	{
671	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
672	AssertRCReturn(rc, rc);
673	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
674	}
675	return VINF_SUCCESS;
676	}
677
678
679	/**
680	* Reads the VM-exit interruption error code from the VMCS into the VMX
681	* transient structure.
682	*
683	* @returns VBox status code.
684	* @param pVmxTransient Pointer to the VMX transient structure.
685	*/
686	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
687	{
688	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
689	{
690	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
691	AssertRCReturn(rc, rc);
692	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
693	}
694	return VINF_SUCCESS;
695	}
696
697
698	/**
699	* Reads the VM-exit instruction length field from the VMCS into the VMX
700	* transient structure.
701	*
702	* @returns VBox status code.
703	* @param pVmxTransient Pointer to the VMX transient structure.
704	*/
705	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
706	{
707	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
708	{
709	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
710	AssertRCReturn(rc, rc);
711	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
712	}
713	return VINF_SUCCESS;
714	}
715
716
717	/**
718	* Reads the VM-exit instruction-information field from the VMCS into
719	* the VMX transient structure.
720	*
721	* @returns VBox status code.
722	* @param pVmxTransient Pointer to the VMX transient structure.
723	*/
724	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
725	{
726	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
727	{
728	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
729	AssertRCReturn(rc, rc);
730	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
731	}
732	return VINF_SUCCESS;
733	}
734
735
736	/**
737	* Reads the exit code qualification from the VMCS into the VMX transient
738	* structure.
739	*
740	* @returns VBox status code.
741	* @param pVCpu The cross context virtual CPU structure of the
742	* calling EMT. (Required for the VMCS cache case.)
743	* @param pVmxTransient Pointer to the VMX transient structure.
744	*/
745	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
746	{
747	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
748	{
749	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
750	AssertRCReturn(rc, rc);
751	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
752	}
753	return VINF_SUCCESS;
754	}
755
756
757	/**
758	* Reads the IDT-vectoring information field from the VMCS into the VMX
759	* transient structure.
760	*
761	* @returns VBox status code.
762	* @param pVmxTransient Pointer to the VMX transient structure.
763	*
764	* @remarks No-long-jump zone!!!
765	*/
766	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
767	{
768	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
769	{
770	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
771	AssertRCReturn(rc, rc);
772	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
773	}
774	return VINF_SUCCESS;
775	}
776
777
778	/**
779	* Reads the IDT-vectoring error code from the VMCS into the VMX
780	* transient structure.
781	*
782	* @returns VBox status code.
783	* @param pVmxTransient Pointer to the VMX transient structure.
784	*/
785	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
786	{
787	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
788	{
789	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
790	AssertRCReturn(rc, rc);
791	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
792	}
793	return VINF_SUCCESS;
794	}
795
796
797	/**
798	* Enters VMX root mode operation on the current CPU.
799	*
800	* @returns VBox status code.
801	* @param pVM The cross context VM structure. Can be
802	* NULL, after a resume.
803	* @param HCPhysCpuPage Physical address of the VMXON region.
804	* @param pvCpuPage Pointer to the VMXON region.
805	*/
806	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
807	{
808	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
809	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
810	Assert(pvCpuPage);
811	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
812
813	if (pVM)
814	{
815	/* Write the VMCS revision dword to the VMXON region. */
816	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
817	}
818
819	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
820	RTCCUINTREG fEFlags = ASMIntDisableFlags();
821
822	/* Enable the VMX bit in CR4 if necessary. */
823	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
824
825	/* Enter VMX root mode. */
826	int rc = VMXEnable(HCPhysCpuPage);
827	if (RT_FAILURE(rc))
828	{
829	if (!(uOldCr4 & X86_CR4_VMXE))
830	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
831
832	if (pVM)
833	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
834	}
835
836	/* Restore interrupts. */
837	ASMSetFlags(fEFlags);
838	return rc;
839	}
840
841
842	/**
843	* Exits VMX root mode operation on the current CPU.
844	*
845	* @returns VBox status code.
846	*/
847	static int hmR0VmxLeaveRootMode(void)
848	{
849	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
850
851	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
852	RTCCUINTREG fEFlags = ASMIntDisableFlags();
853
854	/* If we're for some reason not in VMX root mode, then don't leave it. */
855	RTCCUINTREG uHostCR4 = ASMGetCR4();
856
857	int rc;
858	if (uHostCR4 & X86_CR4_VMXE)
859	{
860	/* Exit VMX root mode and clear the VMX bit in CR4. */
861	VMXDisable();
862	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
863	rc = VINF_SUCCESS;
864	}
865	else
866	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
867
868	/* Restore interrupts. */
869	ASMSetFlags(fEFlags);
870	return rc;
871	}
872
873
874	/**
875	* Allocates and maps one physically contiguous page. The allocated page is
876	* zero'd out. (Used by various VT-x structures).
877	*
878	* @returns IPRT status code.
879	* @param pMemObj Pointer to the ring-0 memory object.
880	* @param ppVirt Where to store the virtual address of the
881	* allocation.
882	* @param pHCPhys Where to store the physical address of the
883	* allocation.
884	*/
885	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
886	{
887	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
888	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
889	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
890
891	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
892	if (RT_FAILURE(rc))
893	return rc;
894	ppVirt = RTR0MemObjAddress(pMemObj);
895	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
896	ASMMemZero32(*ppVirt, PAGE_SIZE);
897	return VINF_SUCCESS;
898	}
899
900
901	/**
902	* Frees and unmaps an allocated physical page.
903	*
904	* @param pMemObj Pointer to the ring-0 memory object.
905	* @param ppVirt Where to re-initialize the virtual address of
906	* allocation as 0.
907	* @param pHCPhys Where to re-initialize the physical address of the
908	* allocation as 0.
909	*/
910	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
911	{
912	AssertPtr(pMemObj);
913	AssertPtr(ppVirt);
914	AssertPtr(pHCPhys);
915	if (*pMemObj != NIL_RTR0MEMOBJ)
916	{
917	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
918	AssertRC(rc);
919	*pMemObj = NIL_RTR0MEMOBJ;
920	*ppVirt = 0;
921	*pHCPhys = 0;
922	}
923	}
924
925
926	/**
927	* Worker function to free VT-x related structures.
928	*
929	* @returns IPRT status code.
930	* @param pVM The cross context VM structure.
931	*/
932	static void hmR0VmxStructsFree(PVM pVM)
933	{
934	for (VMCPUID i = 0; i < pVM->cCpus; i++)
935	{
936	PVMCPU pVCpu = &pVM->aCpus[i];
937	AssertPtr(pVCpu);
938
939	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
940	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
941
942	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
944
945	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
946	}
947
948	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
949	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951	#endif
952	}
953
954
955	/**
956	* Worker function to allocate VT-x related VM structures.
957	*
958	* @returns IPRT status code.
959	* @param pVM The cross context VM structure.
960	*/
961	static int hmR0VmxStructsAlloc(PVM pVM)
962	{
963	/*
964	* Initialize members up-front so we can cleanup properly on allocation failure.
965	*/
966	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
967	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
968	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
969	pVM->hm.s.vmx.HCPhys##a_Name = 0;
970
971	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
972	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
973	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
974	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
975
976	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
977	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
978	#endif
979	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
980
981	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
982	for (VMCPUID i = 0; i < pVM->cCpus; i++)
983	{
984	PVMCPU pVCpu = &pVM->aCpus[i];
985	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
986	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
987	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
988	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
989	}
990	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
991	#undef VMXLOCAL_INIT_VM_MEMOBJ
992
993	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
994	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
995	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
996	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
997
998	/*
999	* Allocate all the VT-x structures.
1000	*/
1001	int rc = VINF_SUCCESS;
1002	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1003	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1004	if (RT_FAILURE(rc))
1005	goto cleanup;
1006	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1007	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1008	#endif
1009
1010	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1011	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1012	{
1013	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1014	&pVM->hm.s.vmx.HCPhysApicAccess);
1015	if (RT_FAILURE(rc))
1016	goto cleanup;
1017	}
1018
1019	/*
1020	* Initialize per-VCPU VT-x structures.
1021	*/
1022	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1023	{
1024	PVMCPU pVCpu = &pVM->aCpus[i];
1025	AssertPtr(pVCpu);
1026
1027	/* Allocate the VM control structure (VMCS). */
1028	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1029	if (RT_FAILURE(rc))
1030	goto cleanup;
1031
1032	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1033	if ( PDMHasApic(pVM)
1034	&& (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1035	{
1036	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1037	NULL /* pR3Ptr /, NULL / pRCPtr */);
1038	if (RT_FAILURE(rc))
1039	goto cleanup;
1040	}
1041
1042	/*
1043	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1044	* transparent accesses of specific MSRs.
1045	*
1046	* If the condition for enabling MSR bitmaps changes here, don't forget to
1047	* update HMAreMsrBitmapsAvailable().
1048	*/
1049	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1050	{
1051	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1052	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1053	if (RT_FAILURE(rc))
1054	goto cleanup;
1055	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1056	}
1057
1058	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1059	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1060	if (RT_FAILURE(rc))
1061	goto cleanup;
1062
1063	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1064	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1065	if (RT_FAILURE(rc))
1066	goto cleanup;
1067	}
1068
1069	return VINF_SUCCESS;
1070
1071	cleanup:
1072	hmR0VmxStructsFree(pVM);
1073	return rc;
1074	}
1075
1076
1077	/**
1078	* Does global VT-x initialization (called during module initialization).
1079	*
1080	* @returns VBox status code.
1081	*/
1082	VMMR0DECL(int) VMXR0GlobalInit(void)
1083	{
1084	#ifdef HMVMX_USE_FUNCTION_TABLE
1085	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1086	# ifdef VBOX_STRICT
1087	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1088	Assert(g_apfnVMExitHandlers[i]);
1089	# endif
1090	#endif
1091	return VINF_SUCCESS;
1092	}
1093
1094
1095	/**
1096	* Does global VT-x termination (called during module termination).
1097	*/
1098	VMMR0DECL(void) VMXR0GlobalTerm()
1099	{
1100	/* Nothing to do currently. */
1101	}
1102
1103
1104	/**
1105	* Sets up and activates VT-x on the current CPU.
1106	*
1107	* @returns VBox status code.
1108	* @param pCpu Pointer to the global CPU info struct.
1109	* @param pVM The cross context VM structure. Can be
1110	* NULL after a host resume operation.
1111	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1112	* fEnabledByHost is @c true).
1113	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1114	* @a fEnabledByHost is @c true).
1115	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1116	* enable VT-x on the host.
1117	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1118	*/
1119	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1120	void *pvMsrs)
1121	{
1122	Assert(pCpu);
1123	Assert(pvMsrs);
1124	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1125
1126	/* Enable VT-x if it's not already enabled by the host. */
1127	if (!fEnabledByHost)
1128	{
1129	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1130	if (RT_FAILURE(rc))
1131	return rc;
1132	}
1133
1134	/*
1135	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1136	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1137	*/
1138	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1139	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1140	{
1141	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1142	pCpu->fFlushAsidBeforeUse = false;
1143	}
1144	else
1145	pCpu->fFlushAsidBeforeUse = true;
1146
1147	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1148	++pCpu->cTlbFlushes;
1149
1150	return VINF_SUCCESS;
1151	}
1152
1153
1154	/**
1155	* Deactivates VT-x on the current CPU.
1156	*
1157	* @returns VBox status code.
1158	* @param pCpu Pointer to the global CPU info struct.
1159	* @param pvCpuPage Pointer to the VMXON region.
1160	* @param HCPhysCpuPage Physical address of the VMXON region.
1161	*
1162	* @remarks This function should never be called when SUPR0EnableVTx() or
1163	* similar was used to enable VT-x on the host.
1164	*/
1165	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1166	{
1167	NOREF(pCpu);
1168	NOREF(pvCpuPage);
1169	NOREF(HCPhysCpuPage);
1170
1171	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1172	return hmR0VmxLeaveRootMode();
1173	}
1174
1175
1176	/**
1177	* Sets the permission bits for the specified MSR in the MSR bitmap.
1178	*
1179	* @param pVCpu The cross context virtual CPU structure.
1180	* @param uMsr The MSR value.
1181	* @param enmRead Whether reading this MSR causes a VM-exit.
1182	* @param enmWrite Whether writing this MSR causes a VM-exit.
1183	*/
1184	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1185	{
1186	int32_t iBit;
1187	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189	/*
1190	* Layout:
1191	* 0x000 - 0x3ff - Low MSR read bits
1192	* 0x400 - 0x7ff - High MSR read bits
1193	* 0x800 - 0xbff - Low MSR write bits
1194	* 0xc00 - 0xfff - High MSR write bits
1195	*/
1196	if (uMsr <= 0x00001FFF)
1197	iBit = uMsr;
1198	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1199	{
1200	iBit = uMsr - UINT32_C(0xC0000000);
1201	pbMsrBitmap += 0x400;
1202	}
1203	else
1204	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1205
1206	Assert(iBit <= 0x1fff);
1207	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1208	ASMBitSet(pbMsrBitmap, iBit);
1209	else
1210	ASMBitClear(pbMsrBitmap, iBit);
1211
1212	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1213	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1214	else
1215	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1216	}
1217
1218
1219	#ifdef VBOX_STRICT
1220	/**
1221	* Gets the permission bits for the specified MSR in the MSR bitmap.
1222	*
1223	* @returns VBox status code.
1224	* @retval VINF_SUCCESS if the specified MSR is found.
1225	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1226	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1227	*
1228	* @param pVCpu The cross context virtual CPU structure.
1229	* @param uMsr The MSR.
1230	* @param penmRead Where to store the read permissions.
1231	* @param penmWrite Where to store the write permissions.
1232	*/
1233	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1234	{
1235	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1236	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1237	int32_t iBit;
1238	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1239
1240	/* See hmR0VmxSetMsrPermission() for the layout. */
1241	if (uMsr <= 0x00001FFF)
1242	iBit = uMsr;
1243	else if ( uMsr >= 0xC0000000
1244	&& uMsr <= 0xC0001FFF)
1245	{
1246	iBit = (uMsr - 0xC0000000);
1247	pbMsrBitmap += 0x400;
1248	}
1249	else
1250	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1251
1252	Assert(iBit <= 0x1fff);
1253	if (ASMBitTest(pbMsrBitmap, iBit))
1254	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1255	else
1256	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1257
1258	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1259	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1260	else
1261	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1262	return VINF_SUCCESS;
1263	}
1264	#endif /* VBOX_STRICT */
1265
1266
1267	/**
1268	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1269	* area.
1270	*
1271	* @returns VBox status code.
1272	* @param pVCpu The cross context virtual CPU structure.
1273	* @param cMsrs The number of MSRs.
1274	*/
1275	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1276	{
1277	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1278	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1279	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1280	{
1281	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1282	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1283	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1284	}
1285
1286	/* Update number of guest MSRs to load/store across the world-switch. */
1287	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1288	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1289
1290	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1291	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1292	AssertRCReturn(rc, rc);
1293
1294	/* Update the VCPU's copy of the MSR count. */
1295	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1296
1297	return VINF_SUCCESS;
1298	}
1299
1300
1301	/**
1302	* Adds a new (or updates the value of an existing) guest/host MSR
1303	* pair to be swapped during the world-switch as part of the
1304	* auto-load/store MSR area in the VMCS.
1305	*
1306	* @returns VBox status code.
1307	* @param pVCpu The cross context virtual CPU structure.
1308	* @param uMsr The MSR.
1309	* @param uGuestMsrValue Value of the guest MSR.
1310	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1311	* necessary.
1312	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1313	* its value was updated. Optional, can be NULL.
1314	*/
1315	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1316	bool *pfAddedAndUpdated)
1317	{
1318	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1319	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1320	uint32_t i;
1321	for (i = 0; i < cMsrs; i++)
1322	{
1323	if (pGuestMsr->u32Msr == uMsr)
1324	break;
1325	pGuestMsr++;
1326	}
1327
1328	bool fAdded = false;
1329	if (i == cMsrs)
1330	{
1331	++cMsrs;
1332	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1333	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1334
1335	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1336	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1337	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1338
1339	fAdded = true;
1340	}
1341
1342	/* Update the MSR values in the auto-load/store MSR area. */
1343	pGuestMsr->u32Msr = uMsr;
1344	pGuestMsr->u64Value = uGuestMsrValue;
1345
1346	/* Create/update the MSR slot in the host MSR area. */
1347	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348	pHostMsr += i;
1349	pHostMsr->u32Msr = uMsr;
1350
1351	/*
1352	* Update the host MSR only when requested by the caller AND when we're
1353	* adding it to the auto-load/store area. Otherwise, it would have been
1354	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1355	*/
1356	bool fUpdatedMsrValue = false;
1357	if ( fAdded
1358	&& fUpdateHostMsr)
1359	{
1360	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1361	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1362	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1363	fUpdatedMsrValue = true;
1364	}
1365
1366	if (pfAddedAndUpdated)
1367	*pfAddedAndUpdated = fUpdatedMsrValue;
1368	return VINF_SUCCESS;
1369	}
1370
1371
1372	/**
1373	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1374	* auto-load/store MSR area in the VMCS.
1375	*
1376	* @returns VBox status code.
1377	* @param pVCpu The cross context virtual CPU structure.
1378	* @param uMsr The MSR.
1379	*/
1380	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1381	{
1382	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1383	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1384	for (uint32_t i = 0; i < cMsrs; i++)
1385	{
1386	/* Find the MSR. */
1387	if (pGuestMsr->u32Msr == uMsr)
1388	{
1389	/* If it's the last MSR, simply reduce the count. */
1390	if (i == cMsrs - 1)
1391	{
1392	--cMsrs;
1393	break;
1394	}
1395
1396	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1397	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1398	pLastGuestMsr += cMsrs - 1;
1399	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1400	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1401
1402	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1403	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404	pLastHostMsr += cMsrs - 1;
1405	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1406	pHostMsr->u64Value = pLastHostMsr->u64Value;
1407	--cMsrs;
1408	break;
1409	}
1410	pGuestMsr++;
1411	}
1412
1413	/* Update the VMCS if the count changed (meaning the MSR was found). */
1414	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1415	{
1416	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1417	AssertRCReturn(rc, rc);
1418
1419	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1420	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1421	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1422
1423	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1424	return VINF_SUCCESS;
1425	}
1426
1427	return VERR_NOT_FOUND;
1428	}
1429
1430
1431	/**
1432	* Checks if the specified guest MSR is part of the auto-load/store area in
1433	* the VMCS.
1434	*
1435	* @returns true if found, false otherwise.
1436	* @param pVCpu The cross context virtual CPU structure.
1437	* @param uMsr The MSR to find.
1438	*/
1439	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1440	{
1441	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1445	{
1446	if (pGuestMsr->u32Msr == uMsr)
1447	return true;
1448	}
1449	return false;
1450	}
1451
1452
1453	/**
1454	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1455	*
1456	* @param pVCpu The cross context virtual CPU structure.
1457	*
1458	* @remarks No-long-jump zone!!!
1459	*/
1460	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1461	{
1462	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1463	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1464	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1465	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1466
1467	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1468	{
1469	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1470
1471	/*
1472	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1473	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1474	*/
1475	if (pHostMsr->u32Msr == MSR_K6_EFER)
1476	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1477	else
1478	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1479	}
1480
1481	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1482	}
1483
1484
1485	/**
1486	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1487	* perform lazy restoration of the host MSRs while leaving VT-x.
1488	*
1489	* @param pVCpu The cross context virtual CPU structure.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496
1497	/*
1498	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1499	*/
1500	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1501	{
1502	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1503	#if HC_ARCH_BITS == 64
1504	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505	{
1506	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1507	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1508	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1509	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1510	}
1511	#endif
1512	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1513	}
1514	}
1515
1516
1517	/**
1518	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1519	* lazily while leaving VT-x.
1520	*
1521	* @returns true if it does, false otherwise.
1522	* @param pVCpu The cross context virtual CPU structure.
1523	* @param uMsr The MSR to check.
1524	*/
1525	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1526	{
1527	NOREF(pVCpu);
1528	#if HC_ARCH_BITS == 64
1529	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1530	{
1531	switch (uMsr)
1532	{
1533	case MSR_K8_LSTAR:
1534	case MSR_K6_STAR:
1535	case MSR_K8_SF_MASK:
1536	case MSR_K8_KERNEL_GS_BASE:
1537	return true;
1538	}
1539	}
1540	#else
1541	RT_NOREF(pVCpu, uMsr);
1542	#endif
1543	return false;
1544	}
1545
1546
1547	/**
1548	* Saves a set of guest MSRs back into the guest-CPU context.
1549	*
1550	* @param pVCpu The cross context virtual CPU structure.
1551	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1552	* out-of-sync. Make sure to update the required fields
1553	* before using them.
1554	*
1555	* @remarks No-long-jump zone!!!
1556	*/
1557	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1558	{
1559	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1560	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1561
1562	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1563	{
1564	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1565	#if HC_ARCH_BITS == 64
1566	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1567	{
1568	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1569	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1570	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1571	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1572	}
1573	#else
1574	NOREF(pMixedCtx);
1575	#endif
1576	}
1577	}
1578
1579
1580	/**
1581	* Loads a set of guests MSRs to allow read/passthru to the guest.
1582	*
1583	* The name of this function is slightly confusing. This function does NOT
1584	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1585	* common prefix for functions dealing with "lazy restoration" of the shared
1586	* MSRs.
1587	*
1588	* @param pVCpu The cross context virtual CPU structure.
1589	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1590	* out-of-sync. Make sure to update the required fields
1591	* before using them.
1592	*
1593	* @remarks No-long-jump zone!!!
1594	*/
1595	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1596	{
1597	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1598	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1599
1600	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1601	#if HC_ARCH_BITS == 64
1602	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1603	{
1604	/*
1605	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1606	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1607	* we can skip a few MSR writes.
1608	*
1609	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1610	* guest MSR values in the guest-CPU context might be different to what's currently
1611	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1612	* CPU, see @bugref{8728}.
1613	*/
1614	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1615	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1616	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1617	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1618	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1619	{
1620	#ifdef VBOX_STRICT
1621	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1622	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1623	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1624	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1625	#endif
1626	}
1627	else
1628	{
1629	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1630	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1631	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1632	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1633	}
1634	}
1635	#else
1636	RT_NOREF(pMixedCtx);
1637	#endif
1638	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1639	}
1640
1641
1642	/**
1643	* Performs lazy restoration of the set of host MSRs if they were previously
1644	* loaded with guest MSR values.
1645	*
1646	* @param pVCpu The cross context virtual CPU structure.
1647	*
1648	* @remarks No-long-jump zone!!!
1649	* @remarks The guest MSRs should have been saved back into the guest-CPU
1650	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1651	*/
1652	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1653	{
1654	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1655	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1656
1657	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1658	{
1659	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1660	#if HC_ARCH_BITS == 64
1661	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1662	{
1663	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1664	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1665	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1666	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1667	}
1668	#endif
1669	}
1670	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1671	}
1672
1673
1674	/**
1675	* Verifies that our cached values of the VMCS controls are all
1676	* consistent with what's actually present in the VMCS.
1677	*
1678	* @returns VBox status code.
1679	* @param pVCpu The cross context virtual CPU structure.
1680	*/
1681	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1682	{
1683	uint32_t u32Val;
1684	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1685	AssertRCReturn(rc, rc);
1686	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1687	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1688
1689	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1690	AssertRCReturn(rc, rc);
1691	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1692	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1693
1694	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1695	AssertRCReturn(rc, rc);
1696	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1697	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1698
1699	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1700	AssertRCReturn(rc, rc);
1701	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1702	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1703
1704	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1705	{
1706	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1707	AssertRCReturn(rc, rc);
1708	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1709	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1710	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1711	}
1712
1713	return VINF_SUCCESS;
1714	}
1715
1716
1717	#ifdef VBOX_STRICT
1718	/**
1719	* Verifies that our cached host EFER value has not changed
1720	* since we cached it.
1721	*
1722	* @param pVCpu The cross context virtual CPU structure.
1723	*/
1724	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1725	{
1726	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1727
1728	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1729	{
1730	uint64_t u64Val;
1731	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1732	AssertRC(rc);
1733
1734	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1735	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1736	}
1737	}
1738
1739
1740	/**
1741	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1742	* VMCS are correct.
1743	*
1744	* @param pVCpu The cross context virtual CPU structure.
1745	*/
1746	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1747	{
1748	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1749
1750	/* Verify MSR counts in the VMCS are what we think it should be. */
1751	uint32_t cMsrs;
1752	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1753	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1754
1755	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1756	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1757
1758	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1759	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1760
1761	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1762	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1763	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1764	{
1765	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1766	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1767	pGuestMsr->u32Msr, cMsrs));
1768
1769	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1770	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1771	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1772
1773	/* Verify that the permissions are as expected in the MSR bitmap. */
1774	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1775	{
1776	VMXMSREXITREAD enmRead;
1777	VMXMSREXITWRITE enmWrite;
1778	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1779	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1780	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1781	{
1782	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1783	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1784	}
1785	else
1786	{
1787	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1788	pGuestMsr->u32Msr, cMsrs));
1789	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1790	pGuestMsr->u32Msr, cMsrs));
1791	}
1792	}
1793	}
1794	}
1795	#endif /* VBOX_STRICT */
1796
1797
1798	/**
1799	* Flushes the TLB using EPT.
1800	*
1801	* @returns VBox status code.
1802	* @param pVCpu The cross context virtual CPU structure of the calling
1803	* EMT. Can be NULL depending on @a enmFlush.
1804	* @param enmFlush Type of flush.
1805	*
1806	* @remarks Caller is responsible for making sure this function is called only
1807	* when NestedPaging is supported and providing @a enmFlush that is
1808	* supported by the CPU.
1809	* @remarks Can be called with interrupts disabled.
1810	*/
1811	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1812	{
1813	uint64_t au64Descriptor[2];
1814	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1815	au64Descriptor[0] = 0;
1816	else
1817	{
1818	Assert(pVCpu);
1819	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1820	}
1821	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1822
1823	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1824	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1825	rc));
1826	if ( RT_SUCCESS(rc)
1827	&& pVCpu)
1828	{
1829	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1830	}
1831	}
1832
1833
1834	/**
1835	* Flushes the TLB using VPID.
1836	*
1837	* @returns VBox status code.
1838	* @param pVM The cross context VM structure.
1839	* @param pVCpu The cross context virtual CPU structure of the calling
1840	* EMT. Can be NULL depending on @a enmFlush.
1841	* @param enmFlush Type of flush.
1842	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1843	* on @a enmFlush).
1844	*
1845	* @remarks Can be called with interrupts disabled.
1846	*/
1847	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1848	{
1849	NOREF(pVM);
1850	AssertPtr(pVM);
1851	Assert(pVM->hm.s.vmx.fVpid);
1852
1853	uint64_t au64Descriptor[2];
1854	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1855	{
1856	au64Descriptor[0] = 0;
1857	au64Descriptor[1] = 0;
1858	}
1859	else
1860	{
1861	AssertPtr(pVCpu);
1862	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1863	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1864	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1865	au64Descriptor[1] = GCPtr;
1866	}
1867
1868	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1869	AssertMsg(rc == VINF_SUCCESS,
1870	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1871	if ( RT_SUCCESS(rc)
1872	&& pVCpu)
1873	{
1874	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1875	}
1876	}
1877
1878
1879	/**
1880	* Invalidates a guest page by guest virtual address. Only relevant for
1881	* EPT/VPID, otherwise there is nothing really to invalidate.
1882	*
1883	* @returns VBox status code.
1884	* @param pVM The cross context VM structure.
1885	* @param pVCpu The cross context virtual CPU structure.
1886	* @param GCVirt Guest virtual address of the page to invalidate.
1887	*/
1888	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1889	{
1890	AssertPtr(pVM);
1891	AssertPtr(pVCpu);
1892	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1893
1894	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1895	if (!fFlushPending)
1896	{
1897	/*
1898	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1899	* See @bugref{6043} and @bugref{6177}.
1900	*
1901	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1902	* function maybe called in a loop with individual addresses.
1903	*/
1904	if (pVM->hm.s.vmx.fVpid)
1905	{
1906	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1907
1908	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1909	/*
1910	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1911	* where executing INVVPID outside 64-bit mode does not flush translations of
1912	* 64-bit linear addresses, see @bugref{6208#c72}.
1913	*/
1914	if (RT_HI_U32(GCVirt))
1915	fVpidFlush = false;
1916	#endif
1917
1918	if (fVpidFlush)
1919	{
1920	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1921	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1922	}
1923	else
1924	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1925	}
1926	else if (pVM->hm.s.fNestedPaging)
1927	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1928	}
1929
1930	return VINF_SUCCESS;
1931	}
1932
1933
1934	/**
1935	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1936	* otherwise there is nothing really to invalidate.
1937	*
1938	* @returns VBox status code.
1939	* @param pVM The cross context VM structure.
1940	* @param pVCpu The cross context virtual CPU structure.
1941	* @param GCPhys Guest physical address of the page to invalidate.
1942	*/
1943	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1944	{
1945	NOREF(pVM); NOREF(GCPhys);
1946	LogFlowFunc(("%RGp\n", GCPhys));
1947
1948	/*
1949	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1950	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1951	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1952	*/
1953	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1954	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1955	return VINF_SUCCESS;
1956	}
1957
1958
1959	/**
1960	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1961	* case where neither EPT nor VPID is supported by the CPU.
1962	*
1963	* @param pVM The cross context VM structure.
1964	* @param pVCpu The cross context virtual CPU structure.
1965	* @param pCpu Pointer to the global HM struct.
1966	*
1967	* @remarks Called with interrupts disabled.
1968	*/
1969	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1970	{
1971	AssertPtr(pVCpu);
1972	AssertPtr(pCpu);
1973	NOREF(pVM);
1974
1975	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1976
1977	Assert(pCpu->idCpu != NIL_RTCPUID);
1978	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1979	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1980	pVCpu->hm.s.fForceTLBFlush = false;
1981	return;
1982	}
1983
1984
1985	/**
1986	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1987	*
1988	* @param pVM The cross context VM structure.
1989	* @param pVCpu The cross context virtual CPU structure.
1990	* @param pCpu Pointer to the global HM CPU struct.
1991	* @remarks All references to "ASID" in this function pertains to "VPID" in
1992	* Intel's nomenclature. The reason is, to avoid confusion in compare
1993	* statements since the host-CPU copies are named "ASID".
1994	*
1995	* @remarks Called with interrupts disabled.
1996	*/
1997	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1998	{
1999	#ifdef VBOX_WITH_STATISTICS
2000	bool fTlbFlushed = false;
2001	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2002	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2003	if (!fTlbFlushed) \
2004	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2005	} while (0)
2006	#else
2007	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2008	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2009	#endif
2010
2011	AssertPtr(pVM);
2012	AssertPtr(pCpu);
2013	AssertPtr(pVCpu);
2014	Assert(pCpu->idCpu != NIL_RTCPUID);
2015
2016	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2017	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2018	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2019
2020	/*
2021	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2022	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2023	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2024	*/
2025	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2026	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2027	{
2028	++pCpu->uCurrentAsid;
2029	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2030	{
2031	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2032	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2033	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2034	}
2035
2036	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2037	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2038	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2039
2040	/*
2041	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2042	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2043	*/
2044	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2045	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2046	HMVMX_SET_TAGGED_TLB_FLUSHED();
2047	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2048	}
2049
2050	/* Check for explicit TLB flushes. */
2051	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2052	{
2053	/*
2054	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2055	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2056	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2057	* but not guest-physical mappings.
2058	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2059	*/
2060	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2061	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2062	HMVMX_SET_TAGGED_TLB_FLUSHED();
2063	}
2064
2065	pVCpu->hm.s.fForceTLBFlush = false;
2066	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2067
2068	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2069	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2070	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2071	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2072	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2073	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2074	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2075	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2076	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2077
2078	/* Update VMCS with the VPID. */
2079	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2080	AssertRC(rc);
2081
2082	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2083	}
2084
2085
2086	/**
2087	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2088	*
2089	* @returns VBox status code.
2090	* @param pVM The cross context VM structure.
2091	* @param pVCpu The cross context virtual CPU structure.
2092	* @param pCpu Pointer to the global HM CPU struct.
2093	*
2094	* @remarks Called with interrupts disabled.
2095	*/
2096	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2097	{
2098	AssertPtr(pVM);
2099	AssertPtr(pVCpu);
2100	AssertPtr(pCpu);
2101	Assert(pCpu->idCpu != NIL_RTCPUID);
2102	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2103	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2104
2105	/*
2106	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2107	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2108	*/
2109	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2110	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2111	{
2112	pVCpu->hm.s.fForceTLBFlush = true;
2113	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2114	}
2115
2116	/* Check for explicit TLB flushes. */
2117	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2118	{
2119	pVCpu->hm.s.fForceTLBFlush = true;
2120	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2121	}
2122
2123	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2124	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2125
2126	if (pVCpu->hm.s.fForceTLBFlush)
2127	{
2128	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2129	pVCpu->hm.s.fForceTLBFlush = false;
2130	}
2131	}
2132
2133
2134	/**
2135	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2136	*
2137	* @returns VBox status code.
2138	* @param pVM The cross context VM structure.
2139	* @param pVCpu The cross context virtual CPU structure.
2140	* @param pCpu Pointer to the global HM CPU struct.
2141	*
2142	* @remarks Called with interrupts disabled.
2143	*/
2144	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2145	{
2146	AssertPtr(pVM);
2147	AssertPtr(pVCpu);
2148	AssertPtr(pCpu);
2149	Assert(pCpu->idCpu != NIL_RTCPUID);
2150	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2151	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2152
2153	/*
2154	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2155	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2156	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2157	*/
2158	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2159	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2160	{
2161	pVCpu->hm.s.fForceTLBFlush = true;
2162	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2163	}
2164
2165	/* Check for explicit TLB flushes. */
2166	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2167	{
2168	/*
2169	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2170	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2171	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2172	*/
2173	pVCpu->hm.s.fForceTLBFlush = true;
2174	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2175	}
2176
2177	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2178	if (pVCpu->hm.s.fForceTLBFlush)
2179	{
2180	++pCpu->uCurrentAsid;
2181	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2182	{
2183	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2184	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2185	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2186	}
2187
2188	pVCpu->hm.s.fForceTLBFlush = false;
2189	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2190	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2191	if (pCpu->fFlushAsidBeforeUse)
2192	{
2193	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2194	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2195	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2196	{
2197	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2198	pCpu->fFlushAsidBeforeUse = false;
2199	}
2200	else
2201	{
2202	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2203	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2204	}
2205	}
2206	}
2207
2208	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2209	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2210	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2211	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2212	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2213	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2214	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2215
2216	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2217	AssertRC(rc);
2218	}
2219
2220
2221	/**
2222	* Flushes the guest TLB entry based on CPU capabilities.
2223	*
2224	* @param pVCpu The cross context virtual CPU structure.
2225	* @param pCpu Pointer to the global HM CPU struct.
2226	*/
2227	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2228	{
2229	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2230	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2231	#endif
2232	PVM pVM = pVCpu->CTX_SUFF(pVM);
2233	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2234	{
2235	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2236	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2237	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2238	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2239	default:
2240	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2241	break;
2242	}
2243
2244	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2245	}
2246
2247
2248	/**
2249	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2250	* TLB entries from the host TLB before VM-entry.
2251	*
2252	* @returns VBox status code.
2253	* @param pVM The cross context VM structure.
2254	*/
2255	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2256	{
2257	/*
2258	* Determine optimal flush type for Nested Paging.
2259	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2260	* guest execution (see hmR3InitFinalizeR0()).
2261	*/
2262	if (pVM->hm.s.fNestedPaging)
2263	{
2264	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2265	{
2266	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2267	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2268	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2269	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2270	else
2271	{
2272	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2273	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2274	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2275	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2276	}
2277
2278	/* Make sure the write-back cacheable memory type for EPT is supported. */
2279	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2280	{
2281	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2282	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2283	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2284	}
2285
2286	/* EPT requires a page-walk length of 4. */
2287	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2288	{
2289	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2290	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2291	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2292	}
2293	}
2294	else
2295	{
2296	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2297	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2298	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2299	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2300	}
2301	}
2302
2303	/*
2304	* Determine optimal flush type for VPID.
2305	*/
2306	if (pVM->hm.s.vmx.fVpid)
2307	{
2308	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2309	{
2310	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2311	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2312	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2313	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2314	else
2315	{
2316	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2317	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2318	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2319	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2320	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2321	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2322	pVM->hm.s.vmx.fVpid = false;
2323	}
2324	}
2325	else
2326	{
2327	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2328	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2329	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2330	pVM->hm.s.vmx.fVpid = false;
2331	}
2332	}
2333
2334	/*
2335	* Setup the handler for flushing tagged-TLBs.
2336	*/
2337	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2338	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2339	else if (pVM->hm.s.fNestedPaging)
2340	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2341	else if (pVM->hm.s.vmx.fVpid)
2342	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2343	else
2344	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2345	return VINF_SUCCESS;
2346	}
2347
2348
2349	/**
2350	* Sets up pin-based VM-execution controls in the VMCS.
2351	*
2352	* @returns VBox status code.
2353	* @param pVM The cross context VM structure.
2354	* @param pVCpu The cross context virtual CPU structure.
2355	*/
2356	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2357	{
2358	AssertPtr(pVM);
2359	AssertPtr(pVCpu);
2360
2361	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2362	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2363
2364	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2365	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2366
2367	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2368	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2369
2370	/* Enable the VMX preemption timer. */
2371	if (pVM->hm.s.vmx.fUsePreemptTimer)
2372	{
2373	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2374	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2375	}
2376
2377	#if 0
2378	/* Enable posted-interrupt processing. */
2379	if (pVM->hm.s.fPostedIntrs)
2380	{
2381	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2382	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2383	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2384	}
2385	#endif
2386
2387	if ((val & zap) != val)
2388	{
2389	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2390	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2391	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2392	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2393	}
2394
2395	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2396	AssertRCReturn(rc, rc);
2397
2398	pVCpu->hm.s.vmx.u32PinCtls = val;
2399	return rc;
2400	}
2401
2402
2403	/**
2404	* Sets up processor-based VM-execution controls in the VMCS.
2405	*
2406	* @returns VBox status code.
2407	* @param pVM The cross context VM structure.
2408	* @param pVCpu The cross context virtual CPU structure.
2409	*/
2410	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2411	{
2412	AssertPtr(pVM);
2413	AssertPtr(pVCpu);
2414
2415	int rc = VERR_INTERNAL_ERROR_5;
2416	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2417	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2418
2419	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2420	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2421	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2422	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2423	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2424	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2425	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2426
2427	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2428	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2429	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2430	{
2431	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2432	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2433	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2434	}
2435
2436	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2437	if (!pVM->hm.s.fNestedPaging)
2438	{
2439	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2440	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2441	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2442	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2443	}
2444
2445	/* Use TPR shadowing if supported by the CPU. */
2446	if ( PDMHasApic(pVM)
2447	&& pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2448	{
2449	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2450	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2451	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2452	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2453	AssertRCReturn(rc, rc);
2454
2455	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2456	/* CR8 writes cause a VM-exit based on TPR threshold. */
2457	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2458	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2459	}
2460	else
2461	{
2462	/*
2463	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2464	* Set this control only for 64-bit guests.
2465	*/
2466	if (pVM->hm.s.fAllow64BitGuests)
2467	{
2468	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2469	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2470	}
2471	}
2472
2473	/* Use MSR-bitmaps if supported by the CPU. */
2474	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2475	{
2476	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2477
2478	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2479	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2480	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2481	AssertRCReturn(rc, rc);
2482
2483	/*
2484	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2485	* automatically using dedicated fields in the VMCS.
2486	*/
2487	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2488	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2489	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2490	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2491	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2492
2493	#if HC_ARCH_BITS == 64
2494	/*
2495	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2496	*/
2497	if (pVM->hm.s.fAllow64BitGuests)
2498	{
2499	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2500	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2501	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2502	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2503	}
2504	#endif
2505	/*
2506	* The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2507	* access (writes need to be executed without exiting, reds will #GP-fault anyway).
2508	*/
2509	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2510	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2511
2512	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2513	}
2514
2515	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2516	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2517	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2518
2519	if ((val & zap) != val)
2520	{
2521	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2522	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2523	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2524	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2525	}
2526
2527	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2528	AssertRCReturn(rc, rc);
2529
2530	pVCpu->hm.s.vmx.u32ProcCtls = val;
2531
2532	/*
2533	* Secondary processor-based VM-execution controls.
2534	*/
2535	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2536	{
2537	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2538	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2539
2540	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2541	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2542
2543	if (pVM->hm.s.fNestedPaging)
2544	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2545
2546	/*
2547	* Enable the INVPCID instruction if supported by the hardware and we expose
2548	* it to the guest. Without this, guest executing INVPCID would cause a #UD.
2549	*/
2550	if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2551	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2552	{
2553	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2554	}
2555
2556	if (pVM->hm.s.vmx.fVpid)
2557	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2558
2559	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2560	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2561
2562	#if 0
2563	if (pVM->hm.s.fVirtApicRegs)
2564	{
2565	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2566	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2567
2568	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2569	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2570	}
2571	#endif
2572
2573	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2574	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2575	* done dynamically. */
2576	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2577	{
2578	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2579	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2580	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2581	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2582	AssertRCReturn(rc, rc);
2583	}
2584
2585	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2586	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2587
2588	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2589	&& pVM->hm.s.vmx.cPleGapTicks
2590	&& pVM->hm.s.vmx.cPleWindowTicks)
2591	{
2592	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2593
2594	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2595	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2596	AssertRCReturn(rc, rc);
2597	}
2598
2599	if ((val & zap) != val)
2600	{
2601	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2602	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2603	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2604	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2605	}
2606
2607	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2608	AssertRCReturn(rc, rc);
2609
2610	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2611	}
2612	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2613	{
2614	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2615	"available\n"));
2616	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2617	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2618	}
2619
2620	return VINF_SUCCESS;
2621	}
2622
2623
2624	/**
2625	* Sets up miscellaneous (everything other than Pin & Processor-based
2626	* VM-execution) control fields in the VMCS.
2627	*
2628	* @returns VBox status code.
2629	* @param pVM The cross context VM structure.
2630	* @param pVCpu The cross context virtual CPU structure.
2631	*/
2632	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2633	{
2634	NOREF(pVM);
2635	AssertPtr(pVM);
2636	AssertPtr(pVCpu);
2637
2638	int rc = VERR_GENERAL_FAILURE;
2639
2640	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2641	#if 0
2642	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2643	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2644	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2645
2646	/*
2647	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2648	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2649	* We thus use the exception bitmap to control it rather than use both.
2650	*/
2651	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2652	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2653
2654	/** @todo Explore possibility of using IO-bitmaps. */
2655	/* All IO & IOIO instructions cause VM-exits. */
2656	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2657	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2658
2659	/* Initialize the MSR-bitmap area. */
2660	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2661	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2662	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2663	AssertRCReturn(rc, rc);
2664	#endif
2665
2666	/* Setup MSR auto-load/store area. */
2667	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2668	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2669	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2670	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2671	AssertRCReturn(rc, rc);
2672
2673	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2674	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2675	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2676	AssertRCReturn(rc, rc);
2677
2678	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2679	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2680	AssertRCReturn(rc, rc);
2681
2682	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2683	#if 0
2684	/* Setup debug controls */
2685	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2686	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2687	AssertRCReturn(rc, rc);
2688	#endif
2689
2690	return rc;
2691	}
2692
2693
2694	/**
2695	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2696	*
2697	* We shall setup those exception intercepts that don't change during the
2698	* lifetime of the VM here. The rest are done dynamically while loading the
2699	* guest state.
2700	*
2701	* @returns VBox status code.
2702	* @param pVM The cross context VM structure.
2703	* @param pVCpu The cross context virtual CPU structure.
2704	*/
2705	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2706	{
2707	AssertPtr(pVM);
2708	AssertPtr(pVCpu);
2709
2710	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2711
2712	uint32_t u32XcptBitmap = 0;
2713
2714	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2715	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2716
2717	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2718	and writes, and because recursive #DBs can cause the CPU hang, we must always
2719	intercept #DB. */
2720	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2721
2722	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2723	if (!pVM->hm.s.fNestedPaging)
2724	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2725
2726	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2727	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2728	AssertRCReturn(rc, rc);
2729	return rc;
2730	}
2731
2732
2733	/**
2734	* Sets up the initial guest-state mask. The guest-state mask is consulted
2735	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2736	* for the nested virtualization case (as it would cause a VM-exit).
2737	*
2738	* @param pVCpu The cross context virtual CPU structure.
2739	*/
2740	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2741	{
2742	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2743	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2744	return VINF_SUCCESS;
2745	}
2746
2747
2748	/**
2749	* Does per-VM VT-x initialization.
2750	*
2751	* @returns VBox status code.
2752	* @param pVM The cross context VM structure.
2753	*/
2754	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2755	{
2756	LogFlowFunc(("pVM=%p\n", pVM));
2757
2758	int rc = hmR0VmxStructsAlloc(pVM);
2759	if (RT_FAILURE(rc))
2760	{
2761	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2762	return rc;
2763	}
2764
2765	return VINF_SUCCESS;
2766	}
2767
2768
2769	/**
2770	* Does per-VM VT-x termination.
2771	*
2772	* @returns VBox status code.
2773	* @param pVM The cross context VM structure.
2774	*/
2775	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2776	{
2777	LogFlowFunc(("pVM=%p\n", pVM));
2778
2779	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2780	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2781	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2782	#endif
2783	hmR0VmxStructsFree(pVM);
2784	return VINF_SUCCESS;
2785	}
2786
2787
2788	/**
2789	* Sets up the VM for execution under VT-x.
2790	* This function is only called once per-VM during initialization.
2791	*
2792	* @returns VBox status code.
2793	* @param pVM The cross context VM structure.
2794	*/
2795	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2796	{
2797	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2798	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2799
2800	LogFlowFunc(("pVM=%p\n", pVM));
2801
2802	/*
2803	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2804	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2805	*/
2806	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2807	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2808	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2809	{
2810	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2811	return VERR_INTERNAL_ERROR;
2812	}
2813
2814	/* Initialize these always, see hmR3InitFinalizeR0().*/
2815	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2816	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2817
2818	/* Setup the tagged-TLB flush handlers. */
2819	int rc = hmR0VmxSetupTaggedTlb(pVM);
2820	if (RT_FAILURE(rc))
2821	{
2822	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2823	return rc;
2824	}
2825
2826	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2827	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2828	#if HC_ARCH_BITS == 64
2829	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2830	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2831	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2832	{
2833	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2834	}
2835	#endif
2836
2837	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2838	RTCCUINTREG uHostCR4 = ASMGetCR4();
2839	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2840	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2841
2842	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2843	{
2844	PVMCPU pVCpu = &pVM->aCpus[i];
2845	AssertPtr(pVCpu);
2846	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2847
2848	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2849	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2850
2851	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2852	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2853	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2854
2855	/* Set revision dword at the beginning of the VMCS structure. */
2856	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2857
2858	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2859	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2860	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2861	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2862
2863	/* Load this VMCS as the current VMCS. */
2864	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2865	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2866	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2867
2868	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2869	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2870	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2871
2872	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2873	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2874	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2875
2876	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2877	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2878	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2879
2880	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2881	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2882	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2883
2884	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2885	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2886	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2887
2888	#if HC_ARCH_BITS == 32
2889	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2890	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2891	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2892	#endif
2893
2894	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2895	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2896	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2897	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2898
2899	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2900
2901	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2902	}
2903
2904	return VINF_SUCCESS;
2905	}
2906
2907
2908	/**
2909	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2910	* the VMCS.
2911	*
2912	* @returns VBox status code.
2913	* @param pVM The cross context VM structure.
2914	* @param pVCpu The cross context virtual CPU structure.
2915	*/
2916	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2917	{
2918	NOREF(pVM); NOREF(pVCpu);
2919
2920	RTCCUINTREG uReg = ASMGetCR0();
2921	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2922	AssertRCReturn(rc, rc);
2923
2924	uReg = ASMGetCR3();
2925	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2926	AssertRCReturn(rc, rc);
2927
2928	uReg = ASMGetCR4();
2929	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2930	AssertRCReturn(rc, rc);
2931	return rc;
2932	}
2933
2934
2935	#if HC_ARCH_BITS == 64
2936	/**
2937	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2938	* requirements. See hmR0VmxSaveHostSegmentRegs().
2939	*/
2940	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2941	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2942	{ \
2943	bool fValidSelector = true; \
2944	if ((selValue) & X86_SEL_LDT) \
2945	{ \
2946	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2947	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2948	} \
2949	if (fValidSelector) \
2950	{ \
2951	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2952	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2953	} \
2954	(selValue) = 0; \
2955	}
2956	#endif
2957
2958
2959	/**
2960	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2961	* the host-state area in the VMCS.
2962	*
2963	* @returns VBox status code.
2964	* @param pVM The cross context VM structure.
2965	* @param pVCpu The cross context virtual CPU structure.
2966	*/
2967	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2968	{
2969	int rc = VERR_INTERNAL_ERROR_5;
2970
2971	#if HC_ARCH_BITS == 64
2972	/*
2973	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2974	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2975	*
2976	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2977	* Was observed booting Solaris10u10 32-bit guest.
2978	*/
2979	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2980	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2981	{
2982	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2983	pVCpu->idCpu));
2984	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2985	}
2986	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2987	#else
2988	RT_NOREF(pVCpu);
2989	#endif
2990
2991	/*
2992	* Host DS, ES, FS and GS segment registers.
2993	*/
2994	#if HC_ARCH_BITS == 64
2995	RTSEL uSelDS = ASMGetDS();
2996	RTSEL uSelES = ASMGetES();
2997	RTSEL uSelFS = ASMGetFS();
2998	RTSEL uSelGS = ASMGetGS();
2999	#else
3000	RTSEL uSelDS = 0;
3001	RTSEL uSelES = 0;
3002	RTSEL uSelFS = 0;
3003	RTSEL uSelGS = 0;
3004	#endif
3005
3006	/*
3007	* Host CS and SS segment registers.
3008	*/
3009	RTSEL uSelCS = ASMGetCS();
3010	RTSEL uSelSS = ASMGetSS();
3011
3012	/*
3013	* Host TR segment register.
3014	*/
3015	RTSEL uSelTR = ASMGetTR();
3016
3017	#if HC_ARCH_BITS == 64
3018	/*
3019	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3020	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3021	*/
3022	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3023	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3024	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3025	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3026	# undef VMXLOCAL_ADJUST_HOST_SEG
3027	#endif
3028
3029	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3030	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3031	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3032	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3033	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3034	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3035	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3036	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3037	Assert(uSelCS);
3038	Assert(uSelTR);
3039
3040	/* Assertion is right but we would not have updated u32ExitCtls yet. */
3041	#if 0
3042	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3043	Assert(uSelSS != 0);
3044	#endif
3045
3046	/* Write these host selector fields into the host-state area in the VMCS. */
3047	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3048	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3049	#if HC_ARCH_BITS == 64
3050	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3051	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3052	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3053	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3054	#else
3055	NOREF(uSelDS);
3056	NOREF(uSelES);
3057	NOREF(uSelFS);
3058	NOREF(uSelGS);
3059	#endif
3060	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3061	AssertRCReturn(rc, rc);
3062
3063	/*
3064	* Host GDTR and IDTR.
3065	*/
3066	RTGDTR Gdtr;
3067	RTIDTR Idtr;
3068	RT_ZERO(Gdtr);
3069	RT_ZERO(Idtr);
3070	ASMGetGDTR(&Gdtr);
3071	ASMGetIDTR(&Idtr);
3072	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3073	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3074	AssertRCReturn(rc, rc);
3075
3076	#if HC_ARCH_BITS == 64
3077	/*
3078	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3079	* maximum limit (0xffff) on every VM-exit.
3080	*/
3081	if (Gdtr.cbGdt != 0xffff)
3082	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3083
3084	/*
3085	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3086	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3087	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3088	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3089	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3090	* hosts where we are pretty sure it won't cause trouble.
3091	*/
3092	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3093	if (Idtr.cbIdt < 0x0fff)
3094	# else
3095	if (Idtr.cbIdt != 0xffff)
3096	# endif
3097	{
3098	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3099	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3100	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3101	}
3102	#endif
3103
3104	/*
3105	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3106	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3107	*/
3108	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3109	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3110	VERR_VMX_INVALID_HOST_STATE);
3111
3112	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3113	#if HC_ARCH_BITS == 64
3114	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3115
3116	/*
3117	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3118	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3119	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3120	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3121	*
3122	* [1] See Intel spec. 3.5 "System Descriptor Types".
3123	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3124	*/
3125	Assert(pDesc->System.u4Type == 11);
3126	if ( pDesc->System.u16LimitLow != 0x67
3127	\|\| pDesc->System.u4LimitHigh)
3128	{
3129	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3130	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3131	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3132	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3133	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3134	}
3135
3136	/*
3137	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3138	*/
3139	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
3140	{
3141	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3142	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3143	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3144	{
3145	/* The GDT is read-only but the writable GDT is available. */
3146	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3147	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3148	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3149	AssertRCReturn(rc, rc);
3150	}
3151	}
3152	#else
3153	NOREF(pVM);
3154	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3155	#endif
3156	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3157	AssertRCReturn(rc, rc);
3158
3159	/*
3160	* Host FS base and GS base.
3161	*/
3162	#if HC_ARCH_BITS == 64
3163	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3164	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3165	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3166	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3167	AssertRCReturn(rc, rc);
3168
3169	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3170	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3171	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3172	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3173	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3174	#endif
3175	return rc;
3176	}
3177
3178
3179	/**
3180	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3181	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3182	* the host after every successful VM-exit.
3183	*
3184	* @returns VBox status code.
3185	* @param pVM The cross context VM structure.
3186	* @param pVCpu The cross context virtual CPU structure.
3187	*
3188	* @remarks No-long-jump zone!!!
3189	*/
3190	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3191	{
3192	NOREF(pVM);
3193
3194	AssertPtr(pVCpu);
3195	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3196
3197	/*
3198	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3199	* rather than swapping them on every VM-entry.
3200	*/
3201	hmR0VmxLazySaveHostMsrs(pVCpu);
3202
3203	/*
3204	* Host Sysenter MSRs.
3205	*/
3206	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3207	#if HC_ARCH_BITS == 32
3208	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3209	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3210	#else
3211	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3212	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3213	#endif
3214	AssertRCReturn(rc, rc);
3215
3216	/*
3217	* Host EFER MSR.
3218	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3219	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3220	*/
3221	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3222	{
3223	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3224	AssertRCReturn(rc, rc);
3225	}
3226
3227	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3228	* hmR0VmxLoadGuestExitCtls() !! */
3229
3230	return rc;
3231	}
3232
3233
3234	/**
3235	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3236	*
3237	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3238	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3239	* hmR0VMxLoadGuestEntryCtls().
3240	*
3241	* @returns true if we need to load guest EFER, false otherwise.
3242	* @param pVCpu The cross context virtual CPU structure.
3243	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3244	* out-of-sync. Make sure to update the required fields
3245	* before using them.
3246	*
3247	* @remarks Requires EFER, CR4.
3248	* @remarks No-long-jump zone!!!
3249	*/
3250	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3251	{
3252	#ifdef HMVMX_ALWAYS_SWAP_EFER
3253	return true;
3254	#endif
3255
3256	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3257	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3258	if (CPUMIsGuestInLongMode(pVCpu))
3259	return false;
3260	#endif
3261
3262	PVM pVM = pVCpu->CTX_SUFF(pVM);
3263	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3264	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3265
3266	/*
3267	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3268	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3269	*/
3270	if ( CPUMIsGuestInLongMode(pVCpu)
3271	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3272	{
3273	return true;
3274	}
3275
3276	/*
3277	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3278	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3279	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3280	*/
3281	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3282	&& (pMixedCtx->cr0 & X86_CR0_PG)
3283	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3284	{
3285	/* Assert that host is PAE capable. */
3286	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3287	return true;
3288	}
3289
3290	/** @todo Check the latest Intel spec. for any other bits,
3291	* like SMEP/SMAP? */
3292	return false;
3293	}
3294
3295
3296	/**
3297	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3298	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3299	* controls".
3300	*
3301	* @returns VBox status code.
3302	* @param pVCpu The cross context virtual CPU structure.
3303	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3304	* out-of-sync. Make sure to update the required fields
3305	* before using them.
3306	*
3307	* @remarks Requires EFER.
3308	* @remarks No-long-jump zone!!!
3309	*/
3310	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3311	{
3312	int rc = VINF_SUCCESS;
3313	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3314	{
3315	PVM pVM = pVCpu->CTX_SUFF(pVM);
3316	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3317	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3318
3319	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3320	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3321
3322	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3323	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3324	{
3325	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3326	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3327	}
3328	else
3329	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3330
3331	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3332	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3333	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3334	{
3335	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3336	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3337	}
3338
3339	/*
3340	* The following should -not- be set (since we're not in SMM mode):
3341	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3342	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3343	*/
3344
3345	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3346	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3347
3348	if ((val & zap) != val)
3349	{
3350	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3351	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3352	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3353	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3354	}
3355
3356	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3357	AssertRCReturn(rc, rc);
3358
3359	pVCpu->hm.s.vmx.u32EntryCtls = val;
3360	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3361	}
3362	return rc;
3363	}
3364
3365
3366	/**
3367	* Sets up the VM-exit controls in the VMCS.
3368	*
3369	* @returns VBox status code.
3370	* @param pVCpu The cross context virtual CPU structure.
3371	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3372	* out-of-sync. Make sure to update the required fields
3373	* before using them.
3374	*
3375	* @remarks Requires EFER.
3376	*/
3377	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3378	{
3379	NOREF(pMixedCtx);
3380
3381	int rc = VINF_SUCCESS;
3382	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3383	{
3384	PVM pVM = pVCpu->CTX_SUFF(pVM);
3385	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3386	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3387
3388	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3389	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3390
3391	/*
3392	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3393	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3394	*/
3395	#if HC_ARCH_BITS == 64
3396	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3397	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3398	#else
3399	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3400	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3401	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3402	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3403	{
3404	/* The switcher returns to long mode, EFER is managed by the switcher. */
3405	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3406	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3407	}
3408	else
3409	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3410	#endif
3411
3412	/* If the newer VMCS fields for managing EFER exists, use it. */
3413	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3414	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3415	{
3416	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3417	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3418	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3419	}
3420
3421	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3422	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3423
3424	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3425	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3426	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3427
3428	if ( pVM->hm.s.vmx.fUsePreemptTimer
3429	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3430	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3431
3432	if ((val & zap) != val)
3433	{
3434	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3435	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3436	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3437	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3438	}
3439
3440	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3441	AssertRCReturn(rc, rc);
3442
3443	pVCpu->hm.s.vmx.u32ExitCtls = val;
3444	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3445	}
3446	return rc;
3447	}
3448
3449
3450	/**
3451	* Sets the TPR threshold in the VMCS.
3452	*
3453	* @returns VBox status code.
3454	* @param pVCpu The cross context virtual CPU structure.
3455	* @param u32TprThreshold The TPR threshold (task-priority class only).
3456	*/
3457	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3458	{
3459	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3460	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3461	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3462	}
3463
3464
3465	/**
3466	* Loads the guest APIC and related state.
3467	*
3468	* @returns VBox status code.
3469	* @param pVCpu The cross context virtual CPU structure.
3470	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3471	* out-of-sync. Make sure to update the required fields
3472	* before using them.
3473	*
3474	* @remarks No-long-jump zone!!!
3475	*/
3476	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3477	{
3478	NOREF(pMixedCtx);
3479
3480	int rc = VINF_SUCCESS;
3481	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_APIC_STATE))
3482	{
3483	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3484	&& APICIsEnabled(pVCpu))
3485	{
3486	/*
3487	* Setup TPR shadowing.
3488	*/
3489	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3490	{
3491	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3492
3493	bool fPendingIntr = false;
3494	uint8_t u8Tpr = 0;
3495	uint8_t u8PendingIntr = 0;
3496	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3497	AssertRCReturn(rc, rc);
3498
3499	/*
3500	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3501	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3502	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3503	*/
3504	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3505	uint32_t u32TprThreshold = 0;
3506	if (fPendingIntr)
3507	{
3508	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3509	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3510	const uint8_t u8TprPriority = u8Tpr >> 4;
3511	if (u8PendingPriority <= u8TprPriority)
3512	u32TprThreshold = u8PendingPriority;
3513	}
3514
3515	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3516	AssertRCReturn(rc, rc);
3517	}
3518	}
3519	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
3520	}
3521
3522	return rc;
3523	}
3524
3525
3526	/**
3527	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3528	*
3529	* @returns Guest's interruptibility-state.
3530	* @param pVCpu The cross context virtual CPU structure.
3531	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3532	* out-of-sync. Make sure to update the required fields
3533	* before using them.
3534	*
3535	* @remarks No-long-jump zone!!!
3536	*/
3537	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3538	{
3539	/*
3540	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3541	*/
3542	uint32_t uIntrState = 0;
3543	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3544	{
3545	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3546	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3547	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3548	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3549	{
3550	if (pMixedCtx->eflags.Bits.u1IF)
3551	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3552	else
3553	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3554	}
3555	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3556	{
3557	/*
3558	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3559	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3560	*/
3561	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3562	}
3563	}
3564
3565	/*
3566	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3567	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3568	* setting this would block host-NMIs and IRET will not clear the blocking.
3569	*
3570	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3571	*/
3572	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3573	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3574	{
3575	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3576	}
3577
3578	return uIntrState;
3579	}
3580
3581
3582	/**
3583	* Loads the guest's interruptibility-state into the guest-state area in the
3584	* VMCS.
3585	*
3586	* @returns VBox status code.
3587	* @param pVCpu The cross context virtual CPU structure.
3588	* @param uIntrState The interruptibility-state to set.
3589	*/
3590	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3591	{
3592	NOREF(pVCpu);
3593	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3594	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3595	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3596	AssertRC(rc);
3597	return rc;
3598	}
3599
3600
3601	/**
3602	* Loads the exception intercepts required for guest execution in the VMCS.
3603	*
3604	* @returns VBox status code.
3605	* @param pVCpu The cross context virtual CPU structure.
3606	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3607	* out-of-sync. Make sure to update the required fields
3608	* before using them.
3609	*/
3610	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3611	{
3612	NOREF(pMixedCtx);
3613	int rc = VINF_SUCCESS;
3614	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
3615	{
3616	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3617	if (pVCpu->hm.s.fGIMTrapXcptUD)
3618	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3619	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3620	else
3621	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3622	#endif
3623
3624	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3625	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3626
3627	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3628	AssertRCReturn(rc, rc);
3629
3630	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3631	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3632	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3633	}
3634	return rc;
3635	}
3636
3637
3638	/**
3639	* Loads the guest's RIP into the guest-state area in the VMCS.
3640	*
3641	* @returns VBox status code.
3642	* @param pVCpu The cross context virtual CPU structure.
3643	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3644	* out-of-sync. Make sure to update the required fields
3645	* before using them.
3646	*
3647	* @remarks No-long-jump zone!!!
3648	*/
3649	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3650	{
3651	int rc = VINF_SUCCESS;
3652	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3653	{
3654	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3655	AssertRCReturn(rc, rc);
3656
3657	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3658	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3659	HMCPU_CF_VALUE(pVCpu)));
3660	}
3661	return rc;
3662	}
3663
3664
3665	/**
3666	* Loads the guest's RSP into the guest-state area in the VMCS.
3667	*
3668	* @returns VBox status code.
3669	* @param pVCpu The cross context virtual CPU structure.
3670	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3671	* out-of-sync. Make sure to update the required fields
3672	* before using them.
3673	*
3674	* @remarks No-long-jump zone!!!
3675	*/
3676	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3677	{
3678	int rc = VINF_SUCCESS;
3679	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3680	{
3681	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3682	AssertRCReturn(rc, rc);
3683
3684	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3685	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3686	}
3687	return rc;
3688	}
3689
3690
3691	/**
3692	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3693	*
3694	* @returns VBox status code.
3695	* @param pVCpu The cross context virtual CPU structure.
3696	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3697	* out-of-sync. Make sure to update the required fields
3698	* before using them.
3699	*
3700	* @remarks No-long-jump zone!!!
3701	*/
3702	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3703	{
3704	int rc = VINF_SUCCESS;
3705	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3706	{
3707	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3708	Let us assert it as such and use 32-bit VMWRITE. */
3709	Assert(!(pMixedCtx->rflags.u64 >> 32));
3710	X86EFLAGS Eflags = pMixedCtx->eflags;
3711	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3712	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3713	* These will never be cleared/set, unless some other part of the VMM
3714	* code is buggy - in which case we're better of finding and fixing
3715	* those bugs than hiding them. */
3716	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3717	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3718	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3719	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3720
3721	/*
3722	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3723	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3724	*/
3725	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3726	{
3727	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3728	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3729	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3730	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3731	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3732	}
3733
3734	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3735	AssertRCReturn(rc, rc);
3736
3737	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3738	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3739	}
3740	return rc;
3741	}
3742
3743
3744	/**
3745	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3746	*
3747	* @returns VBox status code.
3748	* @param pVCpu The cross context virtual CPU structure.
3749	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3750	* out-of-sync. Make sure to update the required fields
3751	* before using them.
3752	*
3753	* @remarks No-long-jump zone!!!
3754	*/
3755	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3756	{
3757	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3758	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3759	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3760	AssertRCReturn(rc, rc);
3761	return rc;
3762	}
3763
3764
3765	/**
3766	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3767	* CR0 is partially shared with the host and we have to consider the FPU bits.
3768	*
3769	* @returns VBox status code.
3770	* @param pVCpu The cross context virtual CPU structure.
3771	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3772	* out-of-sync. Make sure to update the required fields
3773	* before using them.
3774	*
3775	* @remarks No-long-jump zone!!!
3776	*/
3777	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3778	{
3779	/*
3780	* Guest CR0.
3781	* Guest FPU.
3782	*/
3783	int rc = VINF_SUCCESS;
3784	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3785	{
3786	Assert(!(pMixedCtx->cr0 >> 32));
3787	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3788	PVM pVM = pVCpu->CTX_SUFF(pVM);
3789
3790	/* The guest's view (read access) of its CR0 is unblemished. */
3791	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3792	AssertRCReturn(rc, rc);
3793	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3794
3795	/* Setup VT-x's view of the guest CR0. */
3796	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3797	if (pVM->hm.s.fNestedPaging)
3798	{
3799	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3800	{
3801	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3802	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3803	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3804	}
3805	else
3806	{
3807	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3808	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3809	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3810	}
3811
3812	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3813	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3814	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3815
3816	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3817	AssertRCReturn(rc, rc);
3818	}
3819	else
3820	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3821
3822	/*
3823	* Guest FPU bits.
3824	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3825	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3826	*/
3827	u32GuestCR0 \|= X86_CR0_NE;
3828	bool fInterceptNM = false;
3829	if (CPUMIsGuestFPUStateActive(pVCpu))
3830	{
3831	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3832	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3833	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3834	}
3835	else
3836	{
3837	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3838	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3839	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3840	}
3841
3842	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3843	bool fInterceptMF = false;
3844	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3845	fInterceptMF = true;
3846
3847	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3848	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3849	{
3850	Assert(PDMVmmDevHeapIsEnabled(pVM));
3851	Assert(pVM->hm.s.vmx.pRealModeTSS);
3852	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3853	fInterceptNM = true;
3854	fInterceptMF = true;
3855	}
3856	else
3857	{
3858	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3859	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3860	}
3861	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3862
3863	if (fInterceptNM)
3864	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3865	else
3866	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3867
3868	if (fInterceptMF)
3869	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3870	else
3871	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3872
3873	/* Additional intercepts for debugging, define these yourself explicitly. */
3874	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3875	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3876	\| RT_BIT(X86_XCPT_BP)
3877	\| RT_BIT(X86_XCPT_DE)
3878	\| RT_BIT(X86_XCPT_NM)
3879	\| RT_BIT(X86_XCPT_TS)
3880	\| RT_BIT(X86_XCPT_UD)
3881	\| RT_BIT(X86_XCPT_NP)
3882	\| RT_BIT(X86_XCPT_SS)
3883	\| RT_BIT(X86_XCPT_GP)
3884	\| RT_BIT(X86_XCPT_PF)
3885	\| RT_BIT(X86_XCPT_MF)
3886	;
3887	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3888	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3889	#endif
3890
3891	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3892
3893	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3894	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3895	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3896	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3897	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3898	else
3899	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3900
3901	u32GuestCR0 \|= uSetCR0;
3902	u32GuestCR0 &= uZapCR0;
3903	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3904
3905	/* Write VT-x's view of the guest CR0 into the VMCS. */
3906	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3907	AssertRCReturn(rc, rc);
3908	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3909	uZapCR0));
3910
3911	/*
3912	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3913	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3914	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3915	*/
3916	uint32_t u32CR0Mask = 0;
3917	u32CR0Mask = X86_CR0_PE
3918	\| X86_CR0_NE
3919	\| X86_CR0_WP
3920	\| X86_CR0_PG
3921	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3922	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3923	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3924
3925	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3926	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3927	* and @bugref{6944}. */
3928	#if 0
3929	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3930	u32CR0Mask &= ~X86_CR0_PE;
3931	#endif
3932	if (pVM->hm.s.fNestedPaging)
3933	u32CR0Mask &= ~X86_CR0_WP;
3934
3935	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3936	if (fInterceptNM)
3937	{
3938	u32CR0Mask \|= X86_CR0_TS
3939	\| X86_CR0_MP;
3940	}
3941
3942	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3943	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3944	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3945	AssertRCReturn(rc, rc);
3946	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3947
3948	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3949	}
3950	return rc;
3951	}
3952
3953
3954	/**
3955	* Loads the guest control registers (CR3, CR4) into the guest-state area
3956	* in the VMCS.
3957	*
3958	* @returns VBox strict status code.
3959	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3960	* without unrestricted guest access and the VMMDev is not presently
3961	* mapped (e.g. EFI32).
3962	*
3963	* @param pVCpu The cross context virtual CPU structure.
3964	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3965	* out-of-sync. Make sure to update the required fields
3966	* before using them.
3967	*
3968	* @remarks No-long-jump zone!!!
3969	*/
3970	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3971	{
3972	int rc = VINF_SUCCESS;
3973	PVM pVM = pVCpu->CTX_SUFF(pVM);
3974
3975	/*
3976	* Guest CR2.
3977	* It's always loaded in the assembler code. Nothing to do here.
3978	*/
3979
3980	/*
3981	* Guest CR3.
3982	*/
3983	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3984	{
3985	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3986	if (pVM->hm.s.fNestedPaging)
3987	{
3988	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3989
3990	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3991	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3992	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3993	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3994
3995	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3996	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3997	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3998
3999	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4000	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4001	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4002	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
4003	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4004	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4005	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
4006
4007	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
4008	AssertRCReturn(rc, rc);
4009	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
4010
4011	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4012	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
4013	{
4014	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4015	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
4016	{
4017	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4018	AssertRCReturn(rc, rc);
4019	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4020	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4021	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4022	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4023	AssertRCReturn(rc, rc);
4024	}
4025
4026	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
4027	have Unrestricted Execution to handle the guest when it's not using paging. */
4028	GCPhysGuestCR3 = pMixedCtx->cr3;
4029	}
4030	else
4031	{
4032	/*
4033	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4034	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4035	* EPT takes care of translating it to host-physical addresses.
4036	*/
4037	RTGCPHYS GCPhys;
4038	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4039
4040	/* We obtain it here every time as the guest could have relocated this PCI region. */
4041	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4042	if (RT_SUCCESS(rc))
4043	{ /* likely */ }
4044	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4045	{
4046	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4047	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4048	}
4049	else
4050	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4051
4052	GCPhysGuestCR3 = GCPhys;
4053	}
4054
4055	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4056	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4057	}
4058	else
4059	{
4060	/* Non-nested paging case, just use the hypervisor's CR3. */
4061	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4062
4063	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4064	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4065	}
4066	AssertRCReturn(rc, rc);
4067
4068	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4069	}
4070
4071	/*
4072	* Guest CR4.
4073	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4074	*/
4075	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4076	{
4077	Assert(!(pMixedCtx->cr4 >> 32));
4078	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4079
4080	/* The guest's view of its CR4 is unblemished. */
4081	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4082	AssertRCReturn(rc, rc);
4083	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4084
4085	/* Setup VT-x's view of the guest CR4. */
4086	/*
4087	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4088	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4089	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4090	*/
4091	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4092	{
4093	Assert(pVM->hm.s.vmx.pRealModeTSS);
4094	Assert(PDMVmmDevHeapIsEnabled(pVM));
4095	u32GuestCR4 &= ~X86_CR4_VME;
4096	}
4097
4098	if (pVM->hm.s.fNestedPaging)
4099	{
4100	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4101	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4102	{
4103	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4104	u32GuestCR4 \|= X86_CR4_PSE;
4105	/* Our identity mapping is a 32-bit page directory. */
4106	u32GuestCR4 &= ~X86_CR4_PAE;
4107	}
4108	/* else use guest CR4.*/
4109	}
4110	else
4111	{
4112	/*
4113	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4114	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4115	*/
4116	switch (pVCpu->hm.s.enmShadowMode)
4117	{
4118	case PGMMODE_REAL: /* Real-mode. */
4119	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4120	case PGMMODE_32_BIT: /* 32-bit paging. */
4121	{
4122	u32GuestCR4 &= ~X86_CR4_PAE;
4123	break;
4124	}
4125
4126	case PGMMODE_PAE: /* PAE paging. */
4127	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4128	{
4129	u32GuestCR4 \|= X86_CR4_PAE;
4130	break;
4131	}
4132
4133	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4134	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4135	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4136	break;
4137	#endif
4138	default:
4139	AssertFailed();
4140	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4141	}
4142	}
4143
4144	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4145	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4146	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4147	u32GuestCR4 \|= uSetCR4;
4148	u32GuestCR4 &= uZapCR4;
4149
4150	/* Write VT-x's view of the guest CR4 into the VMCS. */
4151	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4152	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4153	AssertRCReturn(rc, rc);
4154
4155	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4156	uint32_t u32CR4Mask = X86_CR4_VME
4157	\| X86_CR4_PAE
4158	\| X86_CR4_PGE
4159	\| X86_CR4_PSE
4160	\| X86_CR4_VMXE;
4161	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4162	u32CR4Mask \|= X86_CR4_OSXSAVE;
4163	if (pVM->cpum.ro.GuestFeatures.fPcid)
4164	u32CR4Mask \|= X86_CR4_PCIDE;
4165	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4166	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4167	AssertRCReturn(rc, rc);
4168
4169	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4170	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4171
4172	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4173	}
4174	return rc;
4175	}
4176
4177
4178	/**
4179	* Loads the guest debug registers into the guest-state area in the VMCS.
4180	*
4181	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4182	*
4183	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4184	*
4185	* @returns VBox status code.
4186	* @param pVCpu The cross context virtual CPU structure.
4187	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4188	* out-of-sync. Make sure to update the required fields
4189	* before using them.
4190	*
4191	* @remarks No-long-jump zone!!!
4192	*/
4193	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4194	{
4195	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4196	return VINF_SUCCESS;
4197
4198	#ifdef VBOX_STRICT
4199	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4200	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4201	{
4202	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4203	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4204	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4205	}
4206	#endif
4207
4208	int rc;
4209	PVM pVM = pVCpu->CTX_SUFF(pVM);
4210	bool fSteppingDB = false;
4211	bool fInterceptMovDRx = false;
4212	if (pVCpu->hm.s.fSingleInstruction)
4213	{
4214	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4215	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4216	{
4217	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4218	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4219	AssertRCReturn(rc, rc);
4220	Assert(fSteppingDB == false);
4221	}
4222	else
4223	{
4224	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4225	pVCpu->hm.s.fClearTrapFlag = true;
4226	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4227	fSteppingDB = true;
4228	}
4229	}
4230
4231	if ( fSteppingDB
4232	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4233	{
4234	/*
4235	* Use the combined guest and host DRx values found in the hypervisor
4236	* register set because the debugger has breakpoints active or someone
4237	* is single stepping on the host side without a monitor trap flag.
4238	*
4239	* Note! DBGF expects a clean DR6 state before executing guest code.
4240	*/
4241	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4242	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4243	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4244	{
4245	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4246	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4247	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4248	}
4249	else
4250	#endif
4251	if (!CPUMIsHyperDebugStateActive(pVCpu))
4252	{
4253	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4254	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4255	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4256	}
4257
4258	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4259	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4260	AssertRCReturn(rc, rc);
4261
4262	pVCpu->hm.s.fUsingHyperDR7 = true;
4263	fInterceptMovDRx = true;
4264	}
4265	else
4266	{
4267	/*
4268	* If the guest has enabled debug registers, we need to load them prior to
4269	* executing guest code so they'll trigger at the right time.
4270	*/
4271	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4272	{
4273	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4274	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4275	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4276	{
4277	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4278	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4279	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4280	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4281	}
4282	else
4283	#endif
4284	if (!CPUMIsGuestDebugStateActive(pVCpu))
4285	{
4286	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4287	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4288	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4289	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4290	}
4291	Assert(!fInterceptMovDRx);
4292	}
4293	/*
4294	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4295	* must intercept #DB in order to maintain a correct DR6 guest value, and
4296	* because we need to intercept it to prevent nested #DBs from hanging the
4297	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4298	*/
4299	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4300	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4301	&& !CPUMIsGuestDebugStateActive(pVCpu))
4302	#else
4303	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4304	#endif
4305	{
4306	fInterceptMovDRx = true;
4307	}
4308
4309	/* Update guest DR7. */
4310	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4311	AssertRCReturn(rc, rc);
4312
4313	pVCpu->hm.s.fUsingHyperDR7 = false;
4314	}
4315
4316	/*
4317	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4318	*/
4319	if (fInterceptMovDRx)
4320	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4321	else
4322	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4323	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4324	AssertRCReturn(rc, rc);
4325
4326	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4327	return VINF_SUCCESS;
4328	}
4329
4330
4331	#ifdef VBOX_STRICT
4332	/**
4333	* Strict function to validate segment registers.
4334	*
4335	* @remarks ASSUMES CR0 is up to date.
4336	*/
4337	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4338	{
4339	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4340	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4341	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4342	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4343	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4344	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4345	{
4346	/* Protected mode checks */
4347	/* CS */
4348	Assert(pCtx->cs.Attr.n.u1Present);
4349	Assert(!(pCtx->cs.Attr.u & 0xf00));
4350	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4351	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4352	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4353	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4354	\|\| (pCtx->cs.Attr.n.u1Granularity));
4355	/* CS cannot be loaded with NULL in protected mode. */
4356	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4357	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4358	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4359	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4360	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4361	else
4362	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4363	/* SS */
4364	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4365	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4366	if ( !(pCtx->cr0 & X86_CR0_PE)
4367	\|\| pCtx->cs.Attr.n.u4Type == 3)
4368	{
4369	Assert(!pCtx->ss.Attr.n.u2Dpl);
4370	}
4371	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4372	{
4373	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4374	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4375	Assert(pCtx->ss.Attr.n.u1Present);
4376	Assert(!(pCtx->ss.Attr.u & 0xf00));
4377	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4378	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4379	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4380	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4381	\|\| (pCtx->ss.Attr.n.u1Granularity));
4382	}
4383	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4384	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4385	{
4386	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4387	Assert(pCtx->ds.Attr.n.u1Present);
4388	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4389	Assert(!(pCtx->ds.Attr.u & 0xf00));
4390	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4391	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4392	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4393	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4394	\|\| (pCtx->ds.Attr.n.u1Granularity));
4395	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4396	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4397	}
4398	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4399	{
4400	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4401	Assert(pCtx->es.Attr.n.u1Present);
4402	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4403	Assert(!(pCtx->es.Attr.u & 0xf00));
4404	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4405	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4406	\|\| !(pCtx->es.Attr.n.u1Granularity));
4407	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4408	\|\| (pCtx->es.Attr.n.u1Granularity));
4409	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4410	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4411	}
4412	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4413	{
4414	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4415	Assert(pCtx->fs.Attr.n.u1Present);
4416	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4417	Assert(!(pCtx->fs.Attr.u & 0xf00));
4418	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4419	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4420	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4421	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4422	\|\| (pCtx->fs.Attr.n.u1Granularity));
4423	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4424	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4425	}
4426	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4427	{
4428	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4429	Assert(pCtx->gs.Attr.n.u1Present);
4430	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4431	Assert(!(pCtx->gs.Attr.u & 0xf00));
4432	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4433	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4434	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4435	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4436	\|\| (pCtx->gs.Attr.n.u1Granularity));
4437	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4438	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4439	}
4440	/* 64-bit capable CPUs. */
4441	# if HC_ARCH_BITS == 64
4442	Assert(!(pCtx->cs.u64Base >> 32));
4443	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4444	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4445	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4446	# endif
4447	}
4448	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4449	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4450	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4451	{
4452	/* Real and v86 mode checks. */
4453	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4454	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4455	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4456	{
4457	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4458	}
4459	else
4460	{
4461	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4462	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4463	}
4464
4465	/* CS */
4466	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4467	Assert(pCtx->cs.u32Limit == 0xffff);
4468	Assert(u32CSAttr == 0xf3);
4469	/* SS */
4470	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4471	Assert(pCtx->ss.u32Limit == 0xffff);
4472	Assert(u32SSAttr == 0xf3);
4473	/* DS */
4474	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4475	Assert(pCtx->ds.u32Limit == 0xffff);
4476	Assert(u32DSAttr == 0xf3);
4477	/* ES */
4478	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4479	Assert(pCtx->es.u32Limit == 0xffff);
4480	Assert(u32ESAttr == 0xf3);
4481	/* FS */
4482	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4483	Assert(pCtx->fs.u32Limit == 0xffff);
4484	Assert(u32FSAttr == 0xf3);
4485	/* GS */
4486	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4487	Assert(pCtx->gs.u32Limit == 0xffff);
4488	Assert(u32GSAttr == 0xf3);
4489	/* 64-bit capable CPUs. */
4490	# if HC_ARCH_BITS == 64
4491	Assert(!(pCtx->cs.u64Base >> 32));
4492	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4493	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4494	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4495	# endif
4496	}
4497	}
4498	#endif /* VBOX_STRICT */
4499
4500
4501	/**
4502	* Writes a guest segment register into the guest-state area in the VMCS.
4503	*
4504	* @returns VBox status code.
4505	* @param pVCpu The cross context virtual CPU structure.
4506	* @param idxSel Index of the selector in the VMCS.
4507	* @param idxLimit Index of the segment limit in the VMCS.
4508	* @param idxBase Index of the segment base in the VMCS.
4509	* @param idxAccess Index of the access rights of the segment in the VMCS.
4510	* @param pSelReg Pointer to the segment selector.
4511	*
4512	* @remarks No-long-jump zone!!!
4513	*/
4514	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4515	uint32_t idxAccess, PCPUMSELREG pSelReg)
4516	{
4517	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4518	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4519	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4520	AssertRCReturn(rc, rc);
4521
4522	uint32_t u32Access = pSelReg->Attr.u;
4523	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4524	{
4525	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4526	u32Access = 0xf3;
4527	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4528	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4529	}
4530	else
4531	{
4532	/*
4533	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4534	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4535	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4536	* loaded in protected-mode have their attribute as 0.
4537	*/
4538	if (!u32Access)
4539	u32Access = X86DESCATTR_UNUSABLE;
4540	}
4541
4542	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4543	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4544	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4545
4546	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4547	AssertRCReturn(rc, rc);
4548	return rc;
4549	}
4550
4551
4552	/**
4553	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4554	* into the guest-state area in the VMCS.
4555	*
4556	* @returns VBox status code.
4557	* @param pVCpu The cross context virtual CPU structure.
4558	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4559	* out-of-sync. Make sure to update the required fields
4560	* before using them.
4561	*
4562	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4563	* @remarks No-long-jump zone!!!
4564	*/
4565	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4566	{
4567	int rc = VERR_INTERNAL_ERROR_5;
4568	PVM pVM = pVCpu->CTX_SUFF(pVM);
4569
4570	/*
4571	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4572	*/
4573	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4574	{
4575	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4576	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4577	{
4578	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4579	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4580	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4581	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4582	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4583	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4584	}
4585
4586	#ifdef VBOX_WITH_REM
4587	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4588	{
4589	Assert(pVM->hm.s.vmx.pRealModeTSS);
4590	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4591	if ( pVCpu->hm.s.vmx.fWasInRealMode
4592	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4593	{
4594	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4595	in real-mode (e.g. OpenBSD 4.0) */
4596	REMFlushTBs(pVM);
4597	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4598	pVCpu->hm.s.vmx.fWasInRealMode = false;
4599	}
4600	}
4601	#endif
4602	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4603	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4604	AssertRCReturn(rc, rc);
4605	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4606	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4607	AssertRCReturn(rc, rc);
4608	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4609	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4610	AssertRCReturn(rc, rc);
4611	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4612	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4613	AssertRCReturn(rc, rc);
4614	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4615	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4616	AssertRCReturn(rc, rc);
4617	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4618	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4619	AssertRCReturn(rc, rc);
4620
4621	#ifdef VBOX_STRICT
4622	/* Validate. */
4623	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4624	#endif
4625
4626	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4627	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4628	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4629	}
4630
4631	/*
4632	* Guest TR.
4633	*/
4634	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4635	{
4636	/*
4637	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4638	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4639	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4640	*/
4641	uint16_t u16Sel = 0;
4642	uint32_t u32Limit = 0;
4643	uint64_t u64Base = 0;
4644	uint32_t u32AccessRights = 0;
4645
4646	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4647	{
4648	u16Sel = pMixedCtx->tr.Sel;
4649	u32Limit = pMixedCtx->tr.u32Limit;
4650	u64Base = pMixedCtx->tr.u64Base;
4651	u32AccessRights = pMixedCtx->tr.Attr.u;
4652	}
4653	else
4654	{
4655	Assert(pVM->hm.s.vmx.pRealModeTSS);
4656	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4657
4658	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4659	RTGCPHYS GCPhys;
4660	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4661	AssertRCReturn(rc, rc);
4662
4663	X86DESCATTR DescAttr;
4664	DescAttr.u = 0;
4665	DescAttr.n.u1Present = 1;
4666	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4667
4668	u16Sel = 0;
4669	u32Limit = HM_VTX_TSS_SIZE;
4670	u64Base = GCPhys; /* in real-mode phys = virt. */
4671	u32AccessRights = DescAttr.u;
4672	}
4673
4674	/* Validate. */
4675	Assert(!(u16Sel & RT_BIT(2)));
4676	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4677	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4678	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4679	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4680	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4681	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4682	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4683	Assert( (u32Limit & 0xfff) == 0xfff
4684	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4685	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4686	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4687
4688	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4689	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4690	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4691	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4692	AssertRCReturn(rc, rc);
4693
4694	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4695	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4696	}
4697
4698	/*
4699	* Guest GDTR.
4700	*/
4701	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4702	{
4703	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4704	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4705	AssertRCReturn(rc, rc);
4706
4707	/* Validate. */
4708	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4709
4710	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4711	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4712	}
4713
4714	/*
4715	* Guest LDTR.
4716	*/
4717	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4718	{
4719	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4720	uint32_t u32Access = 0;
4721	if (!pMixedCtx->ldtr.Attr.u)
4722	u32Access = X86DESCATTR_UNUSABLE;
4723	else
4724	u32Access = pMixedCtx->ldtr.Attr.u;
4725
4726	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4727	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4728	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4729	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4730	AssertRCReturn(rc, rc);
4731
4732	/* Validate. */
4733	if (!(u32Access & X86DESCATTR_UNUSABLE))
4734	{
4735	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4736	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4737	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4738	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4739	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4740	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4741	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4742	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4743	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4744	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4745	}
4746
4747	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4748	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4749	}
4750
4751	/*
4752	* Guest IDTR.
4753	*/
4754	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4755	{
4756	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4757	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4758	AssertRCReturn(rc, rc);
4759
4760	/* Validate. */
4761	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4762
4763	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4764	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4765	}
4766
4767	return VINF_SUCCESS;
4768	}
4769
4770
4771	/**
4772	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4773	* areas.
4774	*
4775	* These MSRs will automatically be loaded to the host CPU on every successful
4776	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4777	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4778	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4779	*
4780	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4781	*
4782	* @returns VBox status code.
4783	* @param pVCpu The cross context virtual CPU structure.
4784	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4785	* out-of-sync. Make sure to update the required fields
4786	* before using them.
4787	*
4788	* @remarks No-long-jump zone!!!
4789	*/
4790	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4791	{
4792	AssertPtr(pVCpu);
4793	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4794
4795	/*
4796	* MSRs that we use the auto-load/store MSR area in the VMCS.
4797	*/
4798	PVM pVM = pVCpu->CTX_SUFF(pVM);
4799	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4800	{
4801	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4802	#if HC_ARCH_BITS == 32
4803	if (pVM->hm.s.fAllow64BitGuests)
4804	{
4805	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4806	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4807	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4808	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4809	AssertRCReturn(rc, rc);
4810	# ifdef LOG_ENABLED
4811	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4812	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4813	{
4814	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4815	pMsr->u64Value));
4816	}
4817	# endif
4818	}
4819	#endif
4820	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4821	}
4822
4823	/*
4824	* Guest Sysenter MSRs.
4825	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4826	* VM-exits on WRMSRs for these MSRs.
4827	*/
4828	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4829	{
4830	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4831	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4832	}
4833
4834	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4835	{
4836	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4837	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4838	}
4839
4840	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4841	{
4842	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4843	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4844	}
4845
4846	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4847	{
4848	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4849	{
4850	/*
4851	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4852	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4853	*/
4854	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4855	{
4856	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4857	AssertRCReturn(rc,rc);
4858	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4859	}
4860	else
4861	{
4862	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4863	NULL /* pfAddedAndUpdated */);
4864	AssertRCReturn(rc, rc);
4865
4866	/* We need to intercept reads too, see @bugref{7386#c16}. */
4867	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4868	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4869	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4870	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4871	}
4872	}
4873	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4874	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4875	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4876	}
4877
4878	return VINF_SUCCESS;
4879	}
4880
4881
4882	/**
4883	* Loads the guest activity state into the guest-state area in the VMCS.
4884	*
4885	* @returns VBox status code.
4886	* @param pVCpu The cross context virtual CPU structure.
4887	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4888	* out-of-sync. Make sure to update the required fields
4889	* before using them.
4890	*
4891	* @remarks No-long-jump zone!!!
4892	*/
4893	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4894	{
4895	NOREF(pMixedCtx);
4896	/** @todo See if we can make use of other states, e.g.
4897	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4898	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4899	{
4900	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4901	AssertRCReturn(rc, rc);
4902
4903	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4904	}
4905	return VINF_SUCCESS;
4906	}
4907
4908
4909	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4910	/**
4911	* Check if guest state allows safe use of 32-bit switcher again.
4912	*
4913	* Segment bases and protected mode structures must be 32-bit addressable
4914	* because the 32-bit switcher will ignore high dword when writing these VMCS
4915	* fields. See @bugref{8432} for details.
4916	*
4917	* @returns true if safe, false if must continue to use the 64-bit switcher.
4918	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4919	* out-of-sync. Make sure to update the required fields
4920	* before using them.
4921	*
4922	* @remarks No-long-jump zone!!!
4923	*/
4924	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4925	{
4926	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4927	return false;
4928	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4929	return false;
4930	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4931	return false;
4932	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4933	return false;
4934	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4935	return false;
4936	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4937	return false;
4938	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4939	return false;
4940	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4941	return false;
4942	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4943	return false;
4944	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4945	return false;
4946	/* All good, bases are 32-bit. */
4947	return true;
4948	}
4949	#endif
4950
4951
4952	/**
4953	* Sets up the appropriate function to run guest code.
4954	*
4955	* @returns VBox status code.
4956	* @param pVCpu The cross context virtual CPU structure.
4957	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4958	* out-of-sync. Make sure to update the required fields
4959	* before using them.
4960	*
4961	* @remarks No-long-jump zone!!!
4962	*/
4963	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4964	{
4965	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4966	{
4967	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4968	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4969	#endif
4970	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4971	#if HC_ARCH_BITS == 32
4972	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4973	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4974	{
4975	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4976	{
4977	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4978	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4979	\| HM_CHANGED_VMX_ENTRY_CTLS
4980	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4981	}
4982	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4983
4984	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4985	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4986	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4987	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4988	}
4989	#else
4990	/* 64-bit host. */
4991	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4992	#endif
4993	}
4994	else
4995	{
4996	/* Guest is not in long mode, use the 32-bit handler. */
4997	#if HC_ARCH_BITS == 32
4998	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4999	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5000	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
5001	{
5002	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5003	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
5004	\| HM_CHANGED_VMX_ENTRY_CTLS
5005	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
5006	}
5007	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5008	/*
5009	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
5010	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
5011	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
5012	* the much faster 32-bit switcher again.
5013	*/
5014	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
5015	{
5016	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
5017	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
5018	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5019	}
5020	else
5021	{
5022	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
5023	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
5024	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
5025	{
5026	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5027	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5028	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5029	\| HM_CHANGED_VMX_ENTRY_CTLS
5030	\| HM_CHANGED_VMX_EXIT_CTLS
5031	\| HM_CHANGED_HOST_CONTEXT);
5032	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5033	}
5034	}
5035	# else
5036	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5037	# endif
5038	#else
5039	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5040	#endif
5041	}
5042	Assert(pVCpu->hm.s.vmx.pfnStartVM);
5043	return VINF_SUCCESS;
5044	}
5045
5046
5047	/**
5048	* Wrapper for running the guest code in VT-x.
5049	*
5050	* @returns VBox status code, no informational status codes.
5051	* @param pVM The cross context VM structure.
5052	* @param pVCpu The cross context virtual CPU structure.
5053	* @param pCtx Pointer to the guest-CPU context.
5054	*
5055	* @remarks No-long-jump zone!!!
5056	*/
5057	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5058	{
5059	/*
5060	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5061	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5062	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5063	*/
5064	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5065	/** @todo Add stats for resume vs launch. */
5066	#ifdef VBOX_WITH_KERNEL_USING_XMM
5067	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5068	#else
5069	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5070	#endif
5071	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5072	return rc;
5073	}
5074
5075
5076	/**
5077	* Reports world-switch error and dumps some useful debug info.
5078	*
5079	* @param pVM The cross context VM structure.
5080	* @param pVCpu The cross context virtual CPU structure.
5081	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5082	* @param pCtx Pointer to the guest-CPU context.
5083	* @param pVmxTransient Pointer to the VMX transient structure (only
5084	* exitReason updated).
5085	*/
5086	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5087	{
5088	Assert(pVM);
5089	Assert(pVCpu);
5090	Assert(pCtx);
5091	Assert(pVmxTransient);
5092	HMVMX_ASSERT_PREEMPT_SAFE();
5093
5094	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5095	switch (rcVMRun)
5096	{
5097	case VERR_VMX_INVALID_VMXON_PTR:
5098	AssertFailed();
5099	break;
5100	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5101	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5102	{
5103	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5104	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5105	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5106	AssertRC(rc);
5107
5108	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5109	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5110	Cannot do it here as we may have been long preempted. */
5111
5112	#ifdef VBOX_STRICT
5113	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5114	pVmxTransient->uExitReason));
5115	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5116	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5117	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5118	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5119	else
5120	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5121	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5122	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5123
5124	/* VMX control bits. */
5125	uint32_t u32Val;
5126	uint64_t u64Val;
5127	RTHCUINTREG uHCReg;
5128	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5129	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5130	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5131	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5132	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5133	{
5134	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5135	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5136	}
5137	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5138	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5139	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5140	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5141	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5142	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5143	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5144	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5145	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5146	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5147	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5148	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5149	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5150	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5151	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5152	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5153	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5154	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5155	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5156	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5157	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5158	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5159	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5160	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5162	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5163	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5164	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5165	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5166	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5167	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5168	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5169	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5170	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5171	if (pVM->hm.s.fNestedPaging)
5172	{
5173	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5174	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5175	}
5176
5177	/* Guest bits. */
5178	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5179	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5180	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5181	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5182	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5183	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5184	if (pVM->hm.s.vmx.fVpid)
5185	{
5186	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5187	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5188	}
5189
5190	/* Host bits. */
5191	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5192	Log4(("Host CR0 %#RHr\n", uHCReg));
5193	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5194	Log4(("Host CR3 %#RHr\n", uHCReg));
5195	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5196	Log4(("Host CR4 %#RHr\n", uHCReg));
5197
5198	RTGDTR HostGdtr;
5199	PCX86DESCHC pDesc;
5200	ASMGetGDTR(&HostGdtr);
5201	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5202	Log4(("Host CS %#08x\n", u32Val));
5203	if (u32Val < HostGdtr.cbGdt)
5204	{
5205	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5206	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5207	}
5208
5209	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5210	Log4(("Host DS %#08x\n", u32Val));
5211	if (u32Val < HostGdtr.cbGdt)
5212	{
5213	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5214	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5215	}
5216
5217	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5218	Log4(("Host ES %#08x\n", u32Val));
5219	if (u32Val < HostGdtr.cbGdt)
5220	{
5221	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5222	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5223	}
5224
5225	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5226	Log4(("Host FS %#08x\n", u32Val));
5227	if (u32Val < HostGdtr.cbGdt)
5228	{
5229	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5230	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5231	}
5232
5233	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5234	Log4(("Host GS %#08x\n", u32Val));
5235	if (u32Val < HostGdtr.cbGdt)
5236	{
5237	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5238	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5239	}
5240
5241	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5242	Log4(("Host SS %#08x\n", u32Val));
5243	if (u32Val < HostGdtr.cbGdt)
5244	{
5245	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5246	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5247	}
5248
5249	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5250	Log4(("Host TR %#08x\n", u32Val));
5251	if (u32Val < HostGdtr.cbGdt)
5252	{
5253	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5254	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5255	}
5256
5257	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5258	Log4(("Host TR Base %#RHv\n", uHCReg));
5259	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5260	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5261	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5262	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5263	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5264	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5265	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5266	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5267	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5268	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5269	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5270	Log4(("Host RSP %#RHv\n", uHCReg));
5271	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5272	Log4(("Host RIP %#RHv\n", uHCReg));
5273	# if HC_ARCH_BITS == 64
5274	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5275	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5276	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5277	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5278	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5279	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5280	# endif
5281	#endif /* VBOX_STRICT */
5282	break;
5283	}
5284
5285	default:
5286	/* Impossible */
5287	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5288	break;
5289	}
5290	NOREF(pVM); NOREF(pCtx);
5291	}
5292
5293
5294	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5295	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5296	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5297	#endif
5298	#ifdef VBOX_STRICT
5299	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5300	{
5301	switch (idxField)
5302	{
5303	case VMX_VMCS_GUEST_RIP:
5304	case VMX_VMCS_GUEST_RSP:
5305	case VMX_VMCS_GUEST_SYSENTER_EIP:
5306	case VMX_VMCS_GUEST_SYSENTER_ESP:
5307	case VMX_VMCS_GUEST_GDTR_BASE:
5308	case VMX_VMCS_GUEST_IDTR_BASE:
5309	case VMX_VMCS_GUEST_CS_BASE:
5310	case VMX_VMCS_GUEST_DS_BASE:
5311	case VMX_VMCS_GUEST_ES_BASE:
5312	case VMX_VMCS_GUEST_FS_BASE:
5313	case VMX_VMCS_GUEST_GS_BASE:
5314	case VMX_VMCS_GUEST_SS_BASE:
5315	case VMX_VMCS_GUEST_LDTR_BASE:
5316	case VMX_VMCS_GUEST_TR_BASE:
5317	case VMX_VMCS_GUEST_CR3:
5318	return true;
5319	}
5320	return false;
5321	}
5322
5323	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5324	{
5325	switch (idxField)
5326	{
5327	/* Read-only fields. */
5328	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5329	return true;
5330	}
5331	/* Remaining readable fields should also be writable. */
5332	return hmR0VmxIsValidWriteField(idxField);
5333	}
5334	#endif /* VBOX_STRICT */
5335
5336
5337	/**
5338	* Executes the specified handler in 64-bit mode.
5339	*
5340	* @returns VBox status code (no informational status codes).
5341	* @param pVM The cross context VM structure.
5342	* @param pVCpu The cross context virtual CPU structure.
5343	* @param pCtx Pointer to the guest CPU context.
5344	* @param enmOp The operation to perform.
5345	* @param cParams Number of parameters.
5346	* @param paParam Array of 32-bit parameters.
5347	*/
5348	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5349	uint32_t cParams, uint32_t *paParam)
5350	{
5351	NOREF(pCtx);
5352
5353	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5354	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5355	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5356	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5357
5358	#ifdef VBOX_STRICT
5359	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5360	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5361
5362	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5363	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5364	#endif
5365
5366	/* Disable interrupts. */
5367	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5368
5369	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5370	RTCPUID idHostCpu = RTMpCpuId();
5371	CPUMR0SetLApic(pVCpu, idHostCpu);
5372	#endif
5373
5374	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5375	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5376
5377	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5378	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5379	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5380
5381	/* Leave VMX Root Mode. */
5382	VMXDisable();
5383
5384	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5385
5386	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5387	CPUMSetHyperEIP(pVCpu, enmOp);
5388	for (int i = (int)cParams - 1; i >= 0; i--)
5389	CPUMPushHyper(pVCpu, paParam[i]);
5390
5391	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5392
5393	/* Call the switcher. */
5394	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5395	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5396
5397	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5398	/* Make sure the VMX instructions don't cause #UD faults. */
5399	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5400
5401	/* Re-enter VMX Root Mode */
5402	int rc2 = VMXEnable(HCPhysCpuPage);
5403	if (RT_FAILURE(rc2))
5404	{
5405	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5406	ASMSetFlags(fOldEFlags);
5407	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5408	return rc2;
5409	}
5410
5411	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5412	AssertRC(rc2);
5413	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5414	Assert(!(ASMGetFlags() & X86_EFL_IF));
5415	ASMSetFlags(fOldEFlags);
5416	return rc;
5417	}
5418
5419
5420	/**
5421	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5422	* supporting 64-bit guests.
5423	*
5424	* @returns VBox status code.
5425	* @param fResume Whether to VMLAUNCH or VMRESUME.
5426	* @param pCtx Pointer to the guest-CPU context.
5427	* @param pCache Pointer to the VMCS cache.
5428	* @param pVM The cross context VM structure.
5429	* @param pVCpu The cross context virtual CPU structure.
5430	*/
5431	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5432	{
5433	NOREF(fResume);
5434
5435	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5436	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5437
5438	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5439	pCache->uPos = 1;
5440	pCache->interPD = PGMGetInterPaeCR3(pVM);
5441	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5442	#endif
5443
5444	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5445	pCache->TestIn.HCPhysCpuPage = 0;
5446	pCache->TestIn.HCPhysVmcs = 0;
5447	pCache->TestIn.pCache = 0;
5448	pCache->TestOut.HCPhysVmcs = 0;
5449	pCache->TestOut.pCache = 0;
5450	pCache->TestOut.pCtx = 0;
5451	pCache->TestOut.eflags = 0;
5452	#else
5453	NOREF(pCache);
5454	#endif
5455
5456	uint32_t aParam[10];
5457	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5458	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5459	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5460	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5461	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5462	aParam[5] = 0;
5463	aParam[6] = VM_RC_ADDR(pVM, pVM);
5464	aParam[7] = 0;
5465	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5466	aParam[9] = 0;
5467
5468	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5469	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5470	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5471	#endif
5472	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5473
5474	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5475	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5476	Assert(pCtx->dr[4] == 10);
5477	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5478	#endif
5479
5480	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5481	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5482	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5483	pVCpu->hm.s.vmx.HCPhysVmcs));
5484	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5485	pCache->TestOut.HCPhysVmcs));
5486	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5487	pCache->TestOut.pCache));
5488	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5489	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5490	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5491	pCache->TestOut.pCtx));
5492	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5493	#endif
5494	return rc;
5495	}
5496
5497
5498	/**
5499	* Initialize the VMCS-Read cache.
5500	*
5501	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5502	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5503	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5504	* (those that have a 32-bit FULL & HIGH part).
5505	*
5506	* @returns VBox status code.
5507	* @param pVM The cross context VM structure.
5508	* @param pVCpu The cross context virtual CPU structure.
5509	*/
5510	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5511	{
5512	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5513	{ \
5514	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5515	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5516	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5517	++cReadFields; \
5518	}
5519
5520	AssertPtr(pVM);
5521	AssertPtr(pVCpu);
5522	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5523	uint32_t cReadFields = 0;
5524
5525	/*
5526	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5527	* and serve to indicate exceptions to the rules.
5528	*/
5529
5530	/* Guest-natural selector base fields. */
5531	#if 0
5532	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5533	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5534	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5535	#endif
5536	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5537	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5538	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5539	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5540	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5541	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5542	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5543	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5544	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5545	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5546	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5547	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5548	#if 0
5549	/* Unused natural width guest-state fields. */
5550	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5551	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5552	#endif
5553	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5554	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5555
5556	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5557	#if 0
5558	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5559	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5560	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5561	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5562	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5563	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5564	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5565	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5566	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5567	#endif
5568
5569	/* Natural width guest-state fields. */
5570	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5571	#if 0
5572	/* Currently unused field. */
5573	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5574	#endif
5575
5576	if (pVM->hm.s.fNestedPaging)
5577	{
5578	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5579	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5580	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5581	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5582	}
5583	else
5584	{
5585	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5586	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5587	}
5588
5589	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5590	return VINF_SUCCESS;
5591	}
5592
5593
5594	/**
5595	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5596	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5597	* darwin, running 64-bit guests).
5598	*
5599	* @returns VBox status code.
5600	* @param pVCpu The cross context virtual CPU structure.
5601	* @param idxField The VMCS field encoding.
5602	* @param u64Val 16, 32 or 64-bit value.
5603	*/
5604	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5605	{
5606	int rc;
5607	switch (idxField)
5608	{
5609	/*
5610	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5611	*/
5612	/* 64-bit Control fields. */
5613	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5614	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5615	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5616	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5617	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5618	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5619	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5620	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5621	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5622	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5623	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5624	case VMX_VMCS64_CTRL_EPTP_FULL:
5625	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5626	/* 64-bit Guest-state fields. */
5627	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5628	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5629	case VMX_VMCS64_GUEST_PAT_FULL:
5630	case VMX_VMCS64_GUEST_EFER_FULL:
5631	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5632	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5633	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5634	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5635	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5636	/* 64-bit Host-state fields. */
5637	case VMX_VMCS64_HOST_PAT_FULL:
5638	case VMX_VMCS64_HOST_EFER_FULL:
5639	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5640	{
5641	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5642	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5643	break;
5644	}
5645
5646	/*
5647	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5648	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5649	*/
5650	/* Natural-width Guest-state fields. */
5651	case VMX_VMCS_GUEST_CR3:
5652	case VMX_VMCS_GUEST_ES_BASE:
5653	case VMX_VMCS_GUEST_CS_BASE:
5654	case VMX_VMCS_GUEST_SS_BASE:
5655	case VMX_VMCS_GUEST_DS_BASE:
5656	case VMX_VMCS_GUEST_FS_BASE:
5657	case VMX_VMCS_GUEST_GS_BASE:
5658	case VMX_VMCS_GUEST_LDTR_BASE:
5659	case VMX_VMCS_GUEST_TR_BASE:
5660	case VMX_VMCS_GUEST_GDTR_BASE:
5661	case VMX_VMCS_GUEST_IDTR_BASE:
5662	case VMX_VMCS_GUEST_RSP:
5663	case VMX_VMCS_GUEST_RIP:
5664	case VMX_VMCS_GUEST_SYSENTER_ESP:
5665	case VMX_VMCS_GUEST_SYSENTER_EIP:
5666	{
5667	if (!(RT_HI_U32(u64Val)))
5668	{
5669	/* If this field is 64-bit, VT-x will zero out the top bits. */
5670	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5671	}
5672	else
5673	{
5674	/* Assert that only the 32->64 switcher case should ever come here. */
5675	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5676	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5677	}
5678	break;
5679	}
5680
5681	default:
5682	{
5683	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5684	rc = VERR_INVALID_PARAMETER;
5685	break;
5686	}
5687	}
5688	AssertRCReturn(rc, rc);
5689	return rc;
5690	}
5691
5692
5693	/**
5694	* Queue up a VMWRITE by using the VMCS write cache.
5695	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5696	*
5697	* @param pVCpu The cross context virtual CPU structure.
5698	* @param idxField The VMCS field encoding.
5699	* @param u64Val 16, 32 or 64-bit value.
5700	*/
5701	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5702	{
5703	AssertPtr(pVCpu);
5704	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5705
5706	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5707	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5708
5709	/* Make sure there are no duplicates. */
5710	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5711	{
5712	if (pCache->Write.aField[i] == idxField)
5713	{
5714	pCache->Write.aFieldVal[i] = u64Val;
5715	return VINF_SUCCESS;
5716	}
5717	}
5718
5719	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5720	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5721	pCache->Write.cValidEntries++;
5722	return VINF_SUCCESS;
5723	}
5724	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5725
5726
5727	/**
5728	* Sets up the usage of TSC-offsetting and updates the VMCS.
5729	*
5730	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5731	* VMX preemption timer.
5732	*
5733	* @returns VBox status code.
5734	* @param pVM The cross context VM structure.
5735	* @param pVCpu The cross context virtual CPU structure.
5736	*
5737	* @remarks No-long-jump zone!!!
5738	*/
5739	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5740	{
5741	int rc;
5742	bool fOffsettedTsc;
5743	bool fParavirtTsc;
5744	if (pVM->hm.s.vmx.fUsePreemptTimer)
5745	{
5746	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5747	&fOffsettedTsc, &fParavirtTsc);
5748
5749	/* Make sure the returned values have sane upper and lower boundaries. */
5750	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5751	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5752	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5753	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5754
5755	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5756	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5757	}
5758	else
5759	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5760
5761	/** @todo later optimize this to be done elsewhere and not before every
5762	* VM-entry. */
5763	if (fParavirtTsc)
5764	{
5765	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5766	information before every VM-entry, hence disable it for performance sake. */
5767	#if 0
5768	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5769	AssertRC(rc);
5770	#endif
5771	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5772	}
5773
5774	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5775	{
5776	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5777	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5778
5779	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5780	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5781	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5782	}
5783	else
5784	{
5785	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5786	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5787	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5788	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5789	}
5790	}
5791
5792
5793	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5794	/**
5795	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5796	* VM-exit interruption info type.
5797	*
5798	* @returns The IEM exception flags.
5799	* @param uVector The event vector.
5800	* @param uVmxVectorType The VMX event type.
5801	*
5802	* @remarks This function currently only constructs flags required for
5803	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5804	* and CR2 aspects of an exception are not included).
5805	*/
5806	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5807	{
5808	uint32_t fIemXcptFlags;
5809	switch (uVmxVectorType)
5810	{
5811	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5812	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5813	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5814	break;
5815
5816	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5817	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5818	break;
5819
5820	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5821	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5822	break;
5823
5824	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5825	{
5826	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5827	if (uVector == X86_XCPT_BP)
5828	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5829	else if (uVector == X86_XCPT_OF)
5830	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5831	else
5832	{
5833	fIemXcptFlags = 0;
5834	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5835	}
5836	break;
5837	}
5838
5839	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5840	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5841	break;
5842
5843	default:
5844	fIemXcptFlags = 0;
5845	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5846	break;
5847	}
5848	return fIemXcptFlags;
5849	}
5850
5851	#else
5852	/**
5853	* Determines if an exception is a contributory exception.
5854	*
5855	* Contributory exceptions are ones which can cause double-faults unless the
5856	* original exception was a benign exception. Page-fault is intentionally not
5857	* included here as it's a conditional contributory exception.
5858	*
5859	* @returns true if the exception is contributory, false otherwise.
5860	* @param uVector The exception vector.
5861	*/
5862	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5863	{
5864	switch (uVector)
5865	{
5866	case X86_XCPT_GP:
5867	case X86_XCPT_SS:
5868	case X86_XCPT_NP:
5869	case X86_XCPT_TS:
5870	case X86_XCPT_DE:
5871	return true;
5872	default:
5873	break;
5874	}
5875	return false;
5876	}
5877	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5878
5879
5880	/**
5881	* Sets an event as a pending event to be injected into the guest.
5882	*
5883	* @param pVCpu The cross context virtual CPU structure.
5884	* @param u32IntInfo The VM-entry interruption-information field.
5885	* @param cbInstr The VM-entry instruction length in bytes (for software
5886	* interrupts, exceptions and privileged software
5887	* exceptions).
5888	* @param u32ErrCode The VM-entry exception error code.
5889	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5890	* page-fault.
5891	*
5892	* @remarks Statistics counter assumes this is a guest event being injected or
5893	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5894	* always incremented.
5895	*/
5896	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5897	RTGCUINTPTR GCPtrFaultAddress)
5898	{
5899	Assert(!pVCpu->hm.s.Event.fPending);
5900	pVCpu->hm.s.Event.fPending = true;
5901	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5902	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5903	pVCpu->hm.s.Event.cbInstr = cbInstr;
5904	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5905	}
5906
5907
5908	/**
5909	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5910	*
5911	* @param pVCpu The cross context virtual CPU structure.
5912	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5913	* out-of-sync. Make sure to update the required fields
5914	* before using them.
5915	*/
5916	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5917	{
5918	NOREF(pMixedCtx);
5919	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5920	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5921	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5922	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5923	}
5924
5925
5926	/**
5927	* Handle a condition that occurred while delivering an event through the guest
5928	* IDT.
5929	*
5930	* @returns Strict VBox status code (i.e. informational status codes too).
5931	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5932	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5933	* to continue execution of the guest which will delivery the \#DF.
5934	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5935	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5936	*
5937	* @param pVCpu The cross context virtual CPU structure.
5938	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5939	* out-of-sync. Make sure to update the required fields
5940	* before using them.
5941	* @param pVmxTransient Pointer to the VMX transient structure.
5942	*
5943	* @remarks No-long-jump zone!!!
5944	*/
5945	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5946	{
5947	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5948
5949	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5950	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5951
5952	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5953	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5954	{
5955	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5956	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5957	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5958	/*
5959	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5960	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5961	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5962	* exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5963	* currently has no way of storing that these exceptions were caused by these instructions
5964	* (ICEBP's #DB poses the problem).
5965	*/
5966	IEMXCPTRAISE enmRaise;
5967	IEMXCPTRAISEINFO fRaiseInfo;
5968	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5969	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5970	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5971	{
5972	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5973	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5974	}
5975	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5976	{
5977	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5978	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5979	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5980	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5981	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5982	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5983	uExitVectorType), VERR_VMX_IPE_5);
5984	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5985
5986	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5987	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5988	{
5989	pVmxTransient->fVectoringPF = true;
5990	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5991	}
5992	}
5993	else
5994	{
5995	/*
5996	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5997	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5998	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5999	*/
6000	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6001	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6002	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6003	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6004	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6005	}
6006
6007	/*
6008	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6009	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6010	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6011	* subsequent VM-entry would fail.
6012	*
6013	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6014	*/
6015	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6016	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6017	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6018	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6019	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6020	{
6021	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6022	}
6023
6024	switch (enmRaise)
6025	{
6026	case IEMXCPTRAISE_CURRENT_XCPT:
6027	{
6028	Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
6029	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6030	Assert(rcStrict == VINF_SUCCESS);
6031	break;
6032	}
6033
6034	case IEMXCPTRAISE_PREV_EVENT:
6035	{
6036	uint32_t u32ErrCode;
6037	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6038	{
6039	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6040	AssertRCReturn(rc2, rc2);
6041	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6042	}
6043	else
6044	u32ErrCode = 0;
6045
6046	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6047	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6048	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6049	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6050
6051	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6052	pVCpu->hm.s.Event.u32ErrCode));
6053	Assert(rcStrict == VINF_SUCCESS);
6054	break;
6055	}
6056
6057	case IEMXCPTRAISE_REEXEC_INSTR:
6058	Assert(rcStrict == VINF_SUCCESS);
6059	break;
6060
6061	case IEMXCPTRAISE_DOUBLE_FAULT:
6062	{
6063	/*
6064	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6065	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6066	*/
6067	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6068	{
6069	pVmxTransient->fVectoringDoublePF = true;
6070	Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6071	pMixedCtx->cr2));
6072	rcStrict = VINF_SUCCESS;
6073	}
6074	else
6075	{
6076	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6077	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6078	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6079	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6080	rcStrict = VINF_HM_DOUBLE_FAULT;
6081	}
6082	break;
6083	}
6084
6085	case IEMXCPTRAISE_TRIPLE_FAULT:
6086	{
6087	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6088	uExitVector));
6089	rcStrict = VINF_EM_RESET;
6090	break;
6091	}
6092
6093	case IEMXCPTRAISE_CPU_HANG:
6094	{
6095	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6096	rcStrict = VERR_EM_GUEST_CPU_HANG;
6097	break;
6098	}
6099
6100	default:
6101	{
6102	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6103	rcStrict = VERR_VMX_IPE_2;
6104	break;
6105	}
6106	}
6107	#else
6108	typedef enum
6109	{
6110	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6111	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6112	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6113	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6114	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6115	} VMXREFLECTXCPT;
6116
6117	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6118	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6119	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6120	{
6121	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6122	{
6123	enmReflect = VMXREFLECTXCPT_XCPT;
6124	#ifdef VBOX_STRICT
6125	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6126	&& uExitVector == X86_XCPT_PF)
6127	{
6128	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6129	}
6130	#endif
6131	if ( uExitVector == X86_XCPT_PF
6132	&& uIdtVector == X86_XCPT_PF)
6133	{
6134	pVmxTransient->fVectoringDoublePF = true;
6135	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6136	}
6137	else if ( uExitVector == X86_XCPT_AC
6138	&& uIdtVector == X86_XCPT_AC)
6139	{
6140	enmReflect = VMXREFLECTXCPT_HANG;
6141	Log4(("IDT: Nested #AC - Bad guest\n"));
6142	}
6143	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6144	&& hmR0VmxIsContributoryXcpt(uExitVector)
6145	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6146	\|\| uIdtVector == X86_XCPT_PF))
6147	{
6148	enmReflect = VMXREFLECTXCPT_DF;
6149	}
6150	else if (uIdtVector == X86_XCPT_DF)
6151	enmReflect = VMXREFLECTXCPT_TF;
6152	}
6153	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6154	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6155	{
6156	/*
6157	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6158	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6159	*/
6160	enmReflect = VMXREFLECTXCPT_XCPT;
6161
6162	if (uExitVector == X86_XCPT_PF)
6163	{
6164	pVmxTransient->fVectoringPF = true;
6165	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6166	}
6167	}
6168	}
6169	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6170	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6171	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6172	{
6173	/*
6174	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6175	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6176	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6177	*/
6178	enmReflect = VMXREFLECTXCPT_XCPT;
6179	}
6180
6181	/*
6182	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6183	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6184	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6185	*
6186	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6187	*/
6188	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6189	&& enmReflect == VMXREFLECTXCPT_XCPT
6190	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6191	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6192	{
6193	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6194	}
6195
6196	switch (enmReflect)
6197	{
6198	case VMXREFLECTXCPT_XCPT:
6199	{
6200	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6201	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6202	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6203
6204	uint32_t u32ErrCode = 0;
6205	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6206	{
6207	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6208	AssertRCReturn(rc2, rc2);
6209	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6210	}
6211
6212	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6213	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6214	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6215	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6216	rcStrict = VINF_SUCCESS;
6217	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6218	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6219
6220	break;
6221	}
6222
6223	case VMXREFLECTXCPT_DF:
6224	{
6225	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6226	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6227	rcStrict = VINF_HM_DOUBLE_FAULT;
6228	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6229	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6230
6231	break;
6232	}
6233
6234	case VMXREFLECTXCPT_TF:
6235	{
6236	rcStrict = VINF_EM_RESET;
6237	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6238	uExitVector));
6239	break;
6240	}
6241
6242	case VMXREFLECTXCPT_HANG:
6243	{
6244	rcStrict = VERR_EM_GUEST_CPU_HANG;
6245	break;
6246	}
6247
6248	default:
6249	Assert(rcStrict == VINF_SUCCESS);
6250	break;
6251	}
6252	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6253	}
6254	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6255	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6256	&& uExitVector != X86_XCPT_DF
6257	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6258	{
6259	/*
6260	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6261	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6262	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6263	*/
6264	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6265	{
6266	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6267	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6268	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6269	}
6270	}
6271
6272	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6273	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6274	return rcStrict;
6275	}
6276
6277
6278	/**
6279	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6280	*
6281	* @returns VBox status code.
6282	* @param pVCpu The cross context virtual CPU structure.
6283	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6284	* out-of-sync. Make sure to update the required fields
6285	* before using them.
6286	*
6287	* @remarks No-long-jump zone!!!
6288	*/
6289	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6290	{
6291	NOREF(pMixedCtx);
6292
6293	/*
6294	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6295	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6296	*/
6297	VMMRZCallRing3Disable(pVCpu);
6298	HM_DISABLE_PREEMPT();
6299
6300	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6301	{
6302	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6303	* and 'dbgc-init' containing:
6304	* sxe "xcpt_de"
6305	* sxe "xcpt_bp"
6306	* sxi "xcpt_gp"
6307	* sxi "xcpt_ss"
6308	* sxi "xcpt_np"
6309	*/
6310	/** @todo r=ramshankar: Should be fixed after r119291. */
6311	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6312	#endif
6313	uint32_t uVal = 0;
6314	uint32_t uShadow = 0;
6315	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6316	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6317	AssertRCReturn(rc, rc);
6318
6319	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6320	CPUMSetGuestCR0(pVCpu, uVal);
6321	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6322	}
6323
6324	HM_RESTORE_PREEMPT();
6325	VMMRZCallRing3Enable(pVCpu);
6326	return VINF_SUCCESS;
6327	}
6328
6329
6330	/**
6331	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6332	*
6333	* @returns VBox status code.
6334	* @param pVCpu The cross context virtual CPU structure.
6335	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6336	* out-of-sync. Make sure to update the required fields
6337	* before using them.
6338	*
6339	* @remarks No-long-jump zone!!!
6340	*/
6341	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6342	{
6343	NOREF(pMixedCtx);
6344
6345	int rc = VINF_SUCCESS;
6346	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6347	{
6348	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6349	uint32_t uVal = 0;
6350	uint32_t uShadow = 0;
6351	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6352	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6353	AssertRCReturn(rc, rc);
6354
6355	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6356	CPUMSetGuestCR4(pVCpu, uVal);
6357	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6358	}
6359	return rc;
6360	}
6361
6362
6363	/**
6364	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6365	*
6366	* @returns VBox status code.
6367	* @param pVCpu The cross context virtual CPU structure.
6368	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6369	* out-of-sync. Make sure to update the required fields
6370	* before using them.
6371	*
6372	* @remarks No-long-jump zone!!!
6373	*/
6374	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6375	{
6376	int rc = VINF_SUCCESS;
6377	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6378	{
6379	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6380	uint64_t u64Val = 0;
6381	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6382	AssertRCReturn(rc, rc);
6383
6384	pMixedCtx->rip = u64Val;
6385	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6386	}
6387	return rc;
6388	}
6389
6390
6391	/**
6392	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6393	*
6394	* @returns VBox status code.
6395	* @param pVCpu The cross context virtual CPU structure.
6396	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6397	* out-of-sync. Make sure to update the required fields
6398	* before using them.
6399	*
6400	* @remarks No-long-jump zone!!!
6401	*/
6402	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6403	{
6404	int rc = VINF_SUCCESS;
6405	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6406	{
6407	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6408	uint64_t u64Val = 0;
6409	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6410	AssertRCReturn(rc, rc);
6411
6412	pMixedCtx->rsp = u64Val;
6413	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6414	}
6415	return rc;
6416	}
6417
6418
6419	/**
6420	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6421	*
6422	* @returns VBox status code.
6423	* @param pVCpu The cross context virtual CPU structure.
6424	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6425	* out-of-sync. Make sure to update the required fields
6426	* before using them.
6427	*
6428	* @remarks No-long-jump zone!!!
6429	*/
6430	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6431	{
6432	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6433	{
6434	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6435	uint32_t uVal = 0;
6436	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6437	AssertRCReturn(rc, rc);
6438
6439	pMixedCtx->eflags.u32 = uVal;
6440	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6441	{
6442	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6443	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6444
6445	pMixedCtx->eflags.Bits.u1VM = 0;
6446	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6447	}
6448
6449	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6450	}
6451	return VINF_SUCCESS;
6452	}
6453
6454
6455	/**
6456	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6457	* guest-CPU context.
6458	*/
6459	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6460	{
6461	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6462	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6463	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6464	return rc;
6465	}
6466
6467
6468	/**
6469	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6470	* from the guest-state area in the VMCS.
6471	*
6472	* @param pVCpu The cross context virtual CPU structure.
6473	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6474	* out-of-sync. Make sure to update the required fields
6475	* before using them.
6476	*
6477	* @remarks No-long-jump zone!!!
6478	*/
6479	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6480	{
6481	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6482	{
6483	uint32_t uIntrState = 0;
6484	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6485	AssertRC(rc);
6486
6487	if (!uIntrState)
6488	{
6489	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6490	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6491
6492	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6493	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6494	}
6495	else
6496	{
6497	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6498	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6499	{
6500	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6501	AssertRC(rc);
6502	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6503	AssertRC(rc);
6504
6505	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6506	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6507	}
6508	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6509	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6510
6511	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6512	{
6513	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6514	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6515	}
6516	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6517	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6518	}
6519
6520	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6521	}
6522	}
6523
6524
6525	/**
6526	* Saves the guest's activity state.
6527	*
6528	* @returns VBox status code.
6529	* @param pVCpu The cross context virtual CPU structure.
6530	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6531	* out-of-sync. Make sure to update the required fields
6532	* before using them.
6533	*
6534	* @remarks No-long-jump zone!!!
6535	*/
6536	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6537	{
6538	NOREF(pMixedCtx);
6539	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6540	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6541	return VINF_SUCCESS;
6542	}
6543
6544
6545	/**
6546	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6547	* the current VMCS into the guest-CPU context.
6548	*
6549	* @returns VBox status code.
6550	* @param pVCpu The cross context virtual CPU structure.
6551	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6552	* out-of-sync. Make sure to update the required fields
6553	* before using them.
6554	*
6555	* @remarks No-long-jump zone!!!
6556	*/
6557	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6558	{
6559	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6560	{
6561	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6562	uint32_t u32Val = 0;
6563	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6564	pMixedCtx->SysEnter.cs = u32Val;
6565	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6566	}
6567
6568	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6569	{
6570	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6571	uint64_t u64Val = 0;
6572	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6573	pMixedCtx->SysEnter.eip = u64Val;
6574	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6575	}
6576	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6577	{
6578	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6579	uint64_t u64Val = 0;
6580	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6581	pMixedCtx->SysEnter.esp = u64Val;
6582	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6583	}
6584	return VINF_SUCCESS;
6585	}
6586
6587
6588	/**
6589	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6590	* the CPU back into the guest-CPU context.
6591	*
6592	* @returns VBox status code.
6593	* @param pVCpu The cross context virtual CPU structure.
6594	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6595	* out-of-sync. Make sure to update the required fields
6596	* before using them.
6597	*
6598	* @remarks No-long-jump zone!!!
6599	*/
6600	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6601	{
6602	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6603	VMMRZCallRing3Disable(pVCpu);
6604	HM_DISABLE_PREEMPT();
6605
6606	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6607	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6608	{
6609	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS));
6610	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6611	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6612	}
6613
6614	HM_RESTORE_PREEMPT();
6615	VMMRZCallRing3Enable(pVCpu);
6616
6617	return VINF_SUCCESS;
6618	}
6619
6620
6621	/**
6622	* Saves the auto load/store'd guest MSRs from the current VMCS into
6623	* the guest-CPU context.
6624	*
6625	* @returns VBox status code.
6626	* @param pVCpu The cross context virtual CPU structure.
6627	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6628	* out-of-sync. Make sure to update the required fields
6629	* before using them.
6630	*
6631	* @remarks No-long-jump zone!!!
6632	*/
6633	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6634	{
6635	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6636	return VINF_SUCCESS;
6637
6638	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6639	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6640	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6641	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6642	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6643	{
6644	switch (pMsr->u32Msr)
6645	{
6646	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6647	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6648	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6649	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6650	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6651	case MSR_IA32_SPEC_CTRL: CPUMR0SetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6652	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6653	break;
6654
6655	default:
6656	{
6657	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6658	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6659	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6660	}
6661	}
6662	}
6663
6664	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6665	return VINF_SUCCESS;
6666	}
6667
6668
6669	/**
6670	* Saves the guest control registers from the current VMCS into the guest-CPU
6671	* context.
6672	*
6673	* @returns VBox status code.
6674	* @param pVCpu The cross context virtual CPU structure.
6675	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6676	* out-of-sync. Make sure to update the required fields
6677	* before using them.
6678	*
6679	* @remarks No-long-jump zone!!!
6680	*/
6681	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6682	{
6683	/* Guest CR0. Guest FPU. */
6684	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6685	AssertRCReturn(rc, rc);
6686
6687	/* Guest CR4. */
6688	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6689	AssertRCReturn(rc, rc);
6690
6691	/* Guest CR2 - updated always during the world-switch or in #PF. */
6692	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6693	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6694	{
6695	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6696	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6697	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6698
6699	PVM pVM = pVCpu->CTX_SUFF(pVM);
6700	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6701	\|\| ( pVM->hm.s.fNestedPaging
6702	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6703	{
6704	uint64_t u64Val = 0;
6705	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6706	if (pMixedCtx->cr3 != u64Val)
6707	{
6708	CPUMSetGuestCR3(pVCpu, u64Val);
6709	if (VMMRZCallRing3IsEnabled(pVCpu))
6710	{
6711	PGMUpdateCR3(pVCpu, u64Val);
6712	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6713	}
6714	else
6715	{
6716	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6717	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6718	}
6719	}
6720
6721	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6722	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6723	{
6724	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6725	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6726	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6727	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6728	AssertRCReturn(rc, rc);
6729
6730	if (VMMRZCallRing3IsEnabled(pVCpu))
6731	{
6732	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6733	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6734	}
6735	else
6736	{
6737	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6738	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6739	}
6740	}
6741	}
6742
6743	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6744	}
6745
6746	/*
6747	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6748	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6749	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6750	*
6751	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6752	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6753	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6754	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6755	*
6756	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6757	*/
6758	if (VMMRZCallRing3IsEnabled(pVCpu))
6759	{
6760	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6761	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6762
6763	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6764	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6765
6766	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6767	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6768	}
6769
6770	return rc;
6771	}
6772
6773
6774	/**
6775	* Reads a guest segment register from the current VMCS into the guest-CPU
6776	* context.
6777	*
6778	* @returns VBox status code.
6779	* @param pVCpu The cross context virtual CPU structure.
6780	* @param idxSel Index of the selector in the VMCS.
6781	* @param idxLimit Index of the segment limit in the VMCS.
6782	* @param idxBase Index of the segment base in the VMCS.
6783	* @param idxAccess Index of the access rights of the segment in the VMCS.
6784	* @param pSelReg Pointer to the segment selector.
6785	*
6786	* @remarks No-long-jump zone!!!
6787	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6788	* macro as that takes care of whether to read from the VMCS cache or
6789	* not.
6790	*/
6791	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6792	PCPUMSELREG pSelReg)
6793	{
6794	NOREF(pVCpu);
6795
6796	uint32_t u32Val = 0;
6797	int rc = VMXReadVmcs32(idxSel, &u32Val);
6798	AssertRCReturn(rc, rc);
6799	pSelReg->Sel = (uint16_t)u32Val;
6800	pSelReg->ValidSel = (uint16_t)u32Val;
6801	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6802
6803	rc = VMXReadVmcs32(idxLimit, &u32Val);
6804	AssertRCReturn(rc, rc);
6805	pSelReg->u32Limit = u32Val;
6806
6807	uint64_t u64Val = 0;
6808	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6809	AssertRCReturn(rc, rc);
6810	pSelReg->u64Base = u64Val;
6811
6812	rc = VMXReadVmcs32(idxAccess, &u32Val);
6813	AssertRCReturn(rc, rc);
6814	pSelReg->Attr.u = u32Val;
6815
6816	/*
6817	* If VT-x marks the segment as unusable, most other bits remain undefined:
6818	* - For CS the L, D and G bits have meaning.
6819	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6820	* - For the remaining data segments no bits are defined.
6821	*
6822	* The present bit and the unusable bit has been observed to be set at the
6823	* same time (the selector was supposed to be invalid as we started executing
6824	* a V8086 interrupt in ring-0).
6825	*
6826	* What should be important for the rest of the VBox code, is that the P bit is
6827	* cleared. Some of the other VBox code recognizes the unusable bit, but
6828	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6829	* safe side here, we'll strip off P and other bits we don't care about. If
6830	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6831	*
6832	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6833	*/
6834	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6835	{
6836	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6837
6838	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6839	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6840	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6841
6842	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6843	#ifdef DEBUG_bird
6844	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6845	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6846	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6847	#endif
6848	}
6849	return VINF_SUCCESS;
6850	}
6851
6852
6853	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6854	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6855	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6856	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6857	#else
6858	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6859	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6860	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6861	#endif
6862
6863
6864	/**
6865	* Saves the guest segment registers from the current VMCS into the guest-CPU
6866	* context.
6867	*
6868	* @returns VBox status code.
6869	* @param pVCpu The cross context virtual CPU structure.
6870	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6871	* out-of-sync. Make sure to update the required fields
6872	* before using them.
6873	*
6874	* @remarks No-long-jump zone!!!
6875	*/
6876	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6877	{
6878	/* Guest segment registers. */
6879	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6880	{
6881	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6882	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6883	AssertRCReturn(rc, rc);
6884
6885	rc = VMXLOCAL_READ_SEG(CS, cs);
6886	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6887	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6888	rc \|= VMXLOCAL_READ_SEG(ES, es);
6889	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6890	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6891	AssertRCReturn(rc, rc);
6892
6893	/* Restore segment attributes for real-on-v86 mode hack. */
6894	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6895	{
6896	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6897	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6898	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6899	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6900	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6901	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6902	}
6903	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6904	}
6905
6906	return VINF_SUCCESS;
6907	}
6908
6909
6910	/**
6911	* Saves the guest descriptor table registers and task register from the current
6912	* VMCS into the guest-CPU context.
6913	*
6914	* @returns VBox status code.
6915	* @param pVCpu The cross context virtual CPU structure.
6916	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6917	* out-of-sync. Make sure to update the required fields
6918	* before using them.
6919	*
6920	* @remarks No-long-jump zone!!!
6921	*/
6922	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6923	{
6924	int rc = VINF_SUCCESS;
6925
6926	/* Guest LDTR. */
6927	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6928	{
6929	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6930	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6931	AssertRCReturn(rc, rc);
6932	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6933	}
6934
6935	/* Guest GDTR. */
6936	uint64_t u64Val = 0;
6937	uint32_t u32Val = 0;
6938	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6939	{
6940	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6941	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6942	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6943	pMixedCtx->gdtr.pGdt = u64Val;
6944	pMixedCtx->gdtr.cbGdt = u32Val;
6945	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6946	}
6947
6948	/* Guest IDTR. */
6949	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6950	{
6951	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6952	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6953	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6954	pMixedCtx->idtr.pIdt = u64Val;
6955	pMixedCtx->idtr.cbIdt = u32Val;
6956	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6957	}
6958
6959	/* Guest TR. */
6960	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6961	{
6962	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6963	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6964	AssertRCReturn(rc, rc);
6965
6966	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6967	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6968	{
6969	rc = VMXLOCAL_READ_SEG(TR, tr);
6970	AssertRCReturn(rc, rc);
6971	}
6972	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6973	}
6974	return rc;
6975	}
6976
6977	#undef VMXLOCAL_READ_SEG
6978
6979
6980	/**
6981	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6982	* context.
6983	*
6984	* @returns VBox status code.
6985	* @param pVCpu The cross context virtual CPU structure.
6986	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6987	* out-of-sync. Make sure to update the required fields
6988	* before using them.
6989	*
6990	* @remarks No-long-jump zone!!!
6991	*/
6992	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6993	{
6994	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6995	{
6996	if (!pVCpu->hm.s.fUsingHyperDR7)
6997	{
6998	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6999	uint32_t u32Val;
7000	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
7001	pMixedCtx->dr[7] = u32Val;
7002	}
7003
7004	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
7005	}
7006	return VINF_SUCCESS;
7007	}
7008
7009
7010	/**
7011	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
7012	*
7013	* @returns VBox status code.
7014	* @param pVCpu The cross context virtual CPU structure.
7015	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7016	* out-of-sync. Make sure to update the required fields
7017	* before using them.
7018	*
7019	* @remarks No-long-jump zone!!!
7020	*/
7021	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7022	{
7023	NOREF(pMixedCtx);
7024
7025	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
7026	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
7027	return VINF_SUCCESS;
7028	}
7029
7030
7031	/**
7032	* Saves the entire guest state from the currently active VMCS into the
7033	* guest-CPU context.
7034	*
7035	* This essentially VMREADs all guest-data.
7036	*
7037	* @returns VBox status code.
7038	* @param pVCpu The cross context virtual CPU structure.
7039	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7040	* out-of-sync. Make sure to update the required fields
7041	* before using them.
7042	*/
7043	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7044	{
7045	Assert(pVCpu);
7046	Assert(pMixedCtx);
7047
7048	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7049	return VINF_SUCCESS;
7050
7051	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7052	again on the ring-3 callback path, there is no real need to. */
7053	if (VMMRZCallRing3IsEnabled(pVCpu))
7054	VMMR0LogFlushDisable(pVCpu);
7055	else
7056	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7057	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7058
7059	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7060	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7061
7062	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7063	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7064
7065	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7066	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7067
7068	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7069	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7070
7071	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7072	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7073
7074	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7075	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7076
7077	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7078	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7079
7080	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7081	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7082
7083	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7084	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7085
7086	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7087	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7088
7089	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7090	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7091	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7092
7093	if (VMMRZCallRing3IsEnabled(pVCpu))
7094	VMMR0LogFlushEnable(pVCpu);
7095
7096	return VINF_SUCCESS;
7097	}
7098
7099
7100	/**
7101	* Saves basic guest registers needed for IEM instruction execution.
7102	*
7103	* @returns VBox status code (OR-able).
7104	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7105	* @param pMixedCtx Pointer to the CPU context of the guest.
7106	* @param fMemory Whether the instruction being executed operates on
7107	* memory or not. Only CR0 is synced up if clear.
7108	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7109	*/
7110	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7111	{
7112	/*
7113	* We assume all general purpose registers other than RSP are available.
7114	*
7115	* - RIP is a must, as it will be incremented or otherwise changed.
7116	* - RFLAGS are always required to figure the CPL.
7117	* - RSP isn't always required, however it's a GPR, so frequently required.
7118	* - SS and CS are the only segment register needed if IEM doesn't do memory
7119	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7120	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7121	* be required for memory accesses.
7122	*
7123	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7124	*/
7125	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7126	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7127	if (fNeedRsp)
7128	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7129	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7130	if (!fMemory)
7131	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7132	else
7133	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7134	AssertRCReturn(rc, rc);
7135	return rc;
7136	}
7137
7138
7139	/**
7140	* Ensures that we've got a complete basic guest-context.
7141	*
7142	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7143	* is for the interpreter.
7144	*
7145	* @returns VBox status code.
7146	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7147	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7148	* needing to be synced in.
7149	* @thread EMT(pVCpu)
7150	*/
7151	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7152	{
7153	/* Note! Since this is only applicable to VT-x, the implementation is placed
7154	in the VT-x part of the sources instead of the generic stuff. */
7155	int rc;
7156	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7157	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7158	else
7159	rc = VINF_SUCCESS;
7160
7161	/*
7162	* For now, imply that the caller might change everything too. Do this after
7163	* saving the guest state so as to not trigger assertions.
7164	*
7165	* This is required for AMD-V too as it too only selectively re-loads changed
7166	* guest state back in to the VMCB.
7167	*/
7168	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7169	return rc;
7170	}
7171
7172
7173	/**
7174	* Check per-VM and per-VCPU force flag actions that require us to go back to
7175	* ring-3 for one reason or another.
7176	*
7177	* @returns Strict VBox status code (i.e. informational status codes too)
7178	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7179	* ring-3.
7180	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7181	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7182	* interrupts)
7183	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7184	* all EMTs to be in ring-3.
7185	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7186	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7187	* to the EM loop.
7188	*
7189	* @param pVM The cross context VM structure.
7190	* @param pVCpu The cross context virtual CPU structure.
7191	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7192	* out-of-sync. Make sure to update the required fields
7193	* before using them.
7194	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7195	*/
7196	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7197	{
7198	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7199
7200	/*
7201	* Anything pending? Should be more likely than not if we're doing a good job.
7202	*/
7203	if ( !fStepping
7204	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7205	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7206	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7207	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7208	return VINF_SUCCESS;
7209
7210	/* We need the control registers now, make sure the guest-CPU context is updated. */
7211	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7212	AssertRCReturn(rc3, rc3);
7213
7214	/* Pending HM CR3 sync. */
7215	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7216	{
7217	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7218	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7219	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7220	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7221	}
7222
7223	/* Pending HM PAE PDPEs. */
7224	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7225	{
7226	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7227	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7228	}
7229
7230	/* Pending PGM C3 sync. */
7231	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7232	{
7233	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7234	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7235	if (rcStrict2 != VINF_SUCCESS)
7236	{
7237	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7238	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7239	return rcStrict2;
7240	}
7241	}
7242
7243	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7244	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7245	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7246	{
7247	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7248	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7249	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7250	return rc2;
7251	}
7252
7253	/* Pending VM request packets, such as hardware interrupts. */
7254	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7255	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7256	{
7257	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7258	return VINF_EM_PENDING_REQUEST;
7259	}
7260
7261	/* Pending PGM pool flushes. */
7262	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7263	{
7264	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7265	return VINF_PGM_POOL_FLUSH_PENDING;
7266	}
7267
7268	/* Pending DMA requests. */
7269	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7270	{
7271	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7272	return VINF_EM_RAW_TO_R3;
7273	}
7274
7275	return VINF_SUCCESS;
7276	}
7277
7278
7279	/**
7280	* Converts any TRPM trap into a pending HM event. This is typically used when
7281	* entering from ring-3 (not longjmp returns).
7282	*
7283	* @param pVCpu The cross context virtual CPU structure.
7284	*/
7285	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7286	{
7287	Assert(TRPMHasTrap(pVCpu));
7288	Assert(!pVCpu->hm.s.Event.fPending);
7289
7290	uint8_t uVector;
7291	TRPMEVENT enmTrpmEvent;
7292	RTGCUINT uErrCode;
7293	RTGCUINTPTR GCPtrFaultAddress;
7294	uint8_t cbInstr;
7295
7296	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7297	AssertRC(rc);
7298
7299	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7300	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7301	if (enmTrpmEvent == TRPM_TRAP)
7302	{
7303	switch (uVector)
7304	{
7305	case X86_XCPT_NMI:
7306	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7307	break;
7308
7309	case X86_XCPT_BP:
7310	case X86_XCPT_OF:
7311	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7312	break;
7313
7314	case X86_XCPT_PF:
7315	case X86_XCPT_DF:
7316	case X86_XCPT_TS:
7317	case X86_XCPT_NP:
7318	case X86_XCPT_SS:
7319	case X86_XCPT_GP:
7320	case X86_XCPT_AC:
7321	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7322	RT_FALL_THRU();
7323	default:
7324	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7325	break;
7326	}
7327	}
7328	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7329	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7330	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7331	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7332	else
7333	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7334
7335	rc = TRPMResetTrap(pVCpu);
7336	AssertRC(rc);
7337	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7338	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7339
7340	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7341	}
7342
7343
7344	/**
7345	* Converts the pending HM event into a TRPM trap.
7346	*
7347	* @param pVCpu The cross context virtual CPU structure.
7348	*/
7349	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7350	{
7351	Assert(pVCpu->hm.s.Event.fPending);
7352
7353	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7354	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7355	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7356	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7357
7358	/* If a trap was already pending, we did something wrong! */
7359	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7360
7361	TRPMEVENT enmTrapType;
7362	switch (uVectorType)
7363	{
7364	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7365	enmTrapType = TRPM_HARDWARE_INT;
7366	break;
7367
7368	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7369	enmTrapType = TRPM_SOFTWARE_INT;
7370	break;
7371
7372	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7373	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7374	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7375	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7376	enmTrapType = TRPM_TRAP;
7377	break;
7378
7379	default:
7380	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7381	enmTrapType = TRPM_32BIT_HACK;
7382	break;
7383	}
7384
7385	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7386
7387	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7388	AssertRC(rc);
7389
7390	if (fErrorCodeValid)
7391	TRPMSetErrorCode(pVCpu, uErrorCode);
7392
7393	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7394	&& uVector == X86_XCPT_PF)
7395	{
7396	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7397	}
7398	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7399	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7400	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7401	{
7402	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7403	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7404	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7405	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7406	}
7407
7408	/* Clear any pending events from the VMCS. */
7409	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7410	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7411
7412	/* We're now done converting the pending event. */
7413	pVCpu->hm.s.Event.fPending = false;
7414	}
7415
7416
7417	/**
7418	* Does the necessary state syncing before returning to ring-3 for any reason
7419	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7420	*
7421	* @returns VBox status code.
7422	* @param pVCpu The cross context virtual CPU structure.
7423	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7424	* be out-of-sync. Make sure to update the required
7425	* fields before using them.
7426	* @param fSaveGuestState Whether to save the guest state or not.
7427	*
7428	* @remarks No-long-jmp zone!!!
7429	*/
7430	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7431	{
7432	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7433	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7434
7435	RTCPUID idCpu = RTMpCpuId();
7436	Log4Func(("HostCpuId=%u\n", idCpu));
7437
7438	/*
7439	* !!! IMPORTANT !!!
7440	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7441	*/
7442
7443	/* Save the guest state if necessary. */
7444	if ( fSaveGuestState
7445	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7446	{
7447	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7448	AssertRCReturn(rc, rc);
7449	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7450	}
7451
7452	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7453	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7454	{
7455	/* We shouldn't reload CR0 without saving it first. */
7456	if (!fSaveGuestState)
7457	{
7458	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7459	AssertRCReturn(rc, rc);
7460	}
7461	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7462	}
7463
7464	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7465	#ifdef VBOX_STRICT
7466	if (CPUMIsHyperDebugStateActive(pVCpu))
7467	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7468	#endif
7469	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7470	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7471	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7472	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7473
7474	#if HC_ARCH_BITS == 64
7475	/* Restore host-state bits that VT-x only restores partially. */
7476	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7477	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7478	{
7479	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7480	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7481	}
7482	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7483	#endif
7484
7485	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7486	if (pVCpu->hm.s.vmx.fLazyMsrs)
7487	{
7488	/* We shouldn't reload the guest MSRs without saving it first. */
7489	if (!fSaveGuestState)
7490	{
7491	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7492	AssertRCReturn(rc, rc);
7493	}
7494	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7495	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7496	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7497	}
7498
7499	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7500	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7501
7502	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7503	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7504	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7505	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7506	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7507	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7508	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7509	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7510
7511	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7512
7513	/** @todo This partially defeats the purpose of having preemption hooks.
7514	* The problem is, deregistering the hooks should be moved to a place that
7515	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7516	* context.
7517	*/
7518	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7519	{
7520	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7521	AssertRCReturn(rc, rc);
7522
7523	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7524	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7525	}
7526	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7527	NOREF(idCpu);
7528
7529	return VINF_SUCCESS;
7530	}
7531
7532
7533	/**
7534	* Leaves the VT-x session.
7535	*
7536	* @returns VBox status code.
7537	* @param pVCpu The cross context virtual CPU structure.
7538	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7539	* out-of-sync. Make sure to update the required fields
7540	* before using them.
7541	*
7542	* @remarks No-long-jmp zone!!!
7543	*/
7544	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7545	{
7546	HM_DISABLE_PREEMPT();
7547	HMVMX_ASSERT_CPU_SAFE();
7548	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7549	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7550
7551	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7552	and done this from the VMXR0ThreadCtxCallback(). */
7553	if (!pVCpu->hm.s.fLeaveDone)
7554	{
7555	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7556	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7557	pVCpu->hm.s.fLeaveDone = true;
7558	}
7559	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7560
7561	/*
7562	* !!! IMPORTANT !!!
7563	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7564	*/
7565
7566	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7567	/** @todo Deregistering here means we need to VMCLEAR always
7568	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7569	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7570	VMMR0ThreadCtxHookDisable(pVCpu);
7571
7572	/* Leave HM context. This takes care of local init (term). */
7573	int rc = HMR0LeaveCpu(pVCpu);
7574
7575	HM_RESTORE_PREEMPT();
7576	return rc;
7577	}
7578
7579
7580	/**
7581	* Does the necessary state syncing before doing a longjmp to ring-3.
7582	*
7583	* @returns VBox status code.
7584	* @param pVCpu The cross context virtual CPU structure.
7585	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7586	* out-of-sync. Make sure to update the required fields
7587	* before using them.
7588	*
7589	* @remarks No-long-jmp zone!!!
7590	*/
7591	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7592	{
7593	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7594	}
7595
7596
7597	/**
7598	* Take necessary actions before going back to ring-3.
7599	*
7600	* An action requires us to go back to ring-3. This function does the necessary
7601	* steps before we can safely return to ring-3. This is not the same as longjmps
7602	* to ring-3, this is voluntary and prepares the guest so it may continue
7603	* executing outside HM (recompiler/IEM).
7604	*
7605	* @returns VBox status code.
7606	* @param pVM The cross context VM structure.
7607	* @param pVCpu The cross context virtual CPU structure.
7608	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7609	* out-of-sync. Make sure to update the required fields
7610	* before using them.
7611	* @param rcExit The reason for exiting to ring-3. Can be
7612	* VINF_VMM_UNKNOWN_RING3_CALL.
7613	*/
7614	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7615	{
7616	Assert(pVM);
7617	Assert(pVCpu);
7618	Assert(pMixedCtx);
7619	HMVMX_ASSERT_PREEMPT_SAFE();
7620
7621	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7622	{
7623	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7624	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7625	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7626	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7627	}
7628
7629	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7630	VMMRZCallRing3Disable(pVCpu);
7631	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7632
7633	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7634	if (pVCpu->hm.s.Event.fPending)
7635	{
7636	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7637	Assert(!pVCpu->hm.s.Event.fPending);
7638	}
7639
7640	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7641	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7642
7643	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7644	and if we're injecting an event we should have a TRPM trap pending. */
7645	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7646	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7647	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7648	#endif
7649
7650	/* Save guest state and restore host state bits. */
7651	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7652	AssertRCReturn(rc, rc);
7653	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7654	/* Thread-context hooks are unregistered at this point!!! */
7655
7656	/* Sync recompiler state. */
7657	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7658	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7659	\| CPUM_CHANGED_LDTR
7660	\| CPUM_CHANGED_GDTR
7661	\| CPUM_CHANGED_IDTR
7662	\| CPUM_CHANGED_TR
7663	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7664	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7665	if ( pVM->hm.s.fNestedPaging
7666	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7667	{
7668	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7669	}
7670
7671	Assert(!pVCpu->hm.s.fClearTrapFlag);
7672
7673	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7674	if (rcExit != VINF_EM_RAW_INTERRUPT)
7675	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7676
7677	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7678
7679	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7680	VMMRZCallRing3RemoveNotification(pVCpu);
7681	VMMRZCallRing3Enable(pVCpu);
7682
7683	return rc;
7684	}
7685
7686
7687	/**
7688	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7689	* longjump to ring-3 and possibly get preempted.
7690	*
7691	* @returns VBox status code.
7692	* @param pVCpu The cross context virtual CPU structure.
7693	* @param enmOperation The operation causing the ring-3 longjump.
7694	* @param pvUser Opaque pointer to the guest-CPU context. The data
7695	* may be out-of-sync. Make sure to update the required
7696	* fields before using them.
7697	*/
7698	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7699	{
7700	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7701	{
7702	/*
7703	* !!! IMPORTANT !!!
7704	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7705	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7706	*/
7707	VMMRZCallRing3RemoveNotification(pVCpu);
7708	VMMRZCallRing3Disable(pVCpu);
7709	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7710	RTThreadPreemptDisable(&PreemptState);
7711
7712	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7713	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7714
7715	#if HC_ARCH_BITS == 64
7716	/* Restore host-state bits that VT-x only restores partially. */
7717	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7718	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7719	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7720	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7721	#endif
7722	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7723	if (pVCpu->hm.s.vmx.fLazyMsrs)
7724	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7725
7726	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7727	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7728	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7729	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7730	{
7731	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7732	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7733	}
7734
7735	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7736	VMMR0ThreadCtxHookDisable(pVCpu);
7737	HMR0LeaveCpu(pVCpu);
7738	RTThreadPreemptRestore(&PreemptState);
7739	return VINF_SUCCESS;
7740	}
7741
7742	Assert(pVCpu);
7743	Assert(pvUser);
7744	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7745	HMVMX_ASSERT_PREEMPT_SAFE();
7746
7747	VMMRZCallRing3Disable(pVCpu);
7748	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7749
7750	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7751	enmOperation));
7752
7753	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7754	AssertRCReturn(rc, rc);
7755
7756	VMMRZCallRing3Enable(pVCpu);
7757	return VINF_SUCCESS;
7758	}
7759
7760
7761	/**
7762	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7763	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7764	*
7765	* @param pVCpu The cross context virtual CPU structure.
7766	*/
7767	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7768	{
7769	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7770	{
7771	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7772	{
7773	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7774	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7775	AssertRC(rc);
7776	Log4(("Setup interrupt-window exiting\n"));
7777	}
7778	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7779	}
7780
7781
7782	/**
7783	* Clears the interrupt-window exiting control in the VMCS.
7784	*
7785	* @param pVCpu The cross context virtual CPU structure.
7786	*/
7787	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7788	{
7789	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7790	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7791	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7792	AssertRC(rc);
7793	Log4(("Cleared interrupt-window exiting\n"));
7794	}
7795
7796
7797	/**
7798	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7799	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7800	*
7801	* @param pVCpu The cross context virtual CPU structure.
7802	*/
7803	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7804	{
7805	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7806	{
7807	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7808	{
7809	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7810	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7811	AssertRC(rc);
7812	Log4(("Setup NMI-window exiting\n"));
7813	}
7814	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7815	}
7816
7817
7818	/**
7819	* Clears the NMI-window exiting control in the VMCS.
7820	*
7821	* @param pVCpu The cross context virtual CPU structure.
7822	*/
7823	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7824	{
7825	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7826	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7827	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7828	AssertRC(rc);
7829	Log4(("Cleared NMI-window exiting\n"));
7830	}
7831
7832
7833	/**
7834	* Evaluates the event to be delivered to the guest and sets it as the pending
7835	* event.
7836	*
7837	* @returns The VT-x guest-interruptibility state.
7838	* @param pVCpu The cross context virtual CPU structure.
7839	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7840	* out-of-sync. Make sure to update the required fields
7841	* before using them.
7842	*/
7843	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7844	{
7845	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7846	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7847	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7848	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7849	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7850
7851	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7852	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7853	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7854	Assert(!TRPMHasTrap(pVCpu));
7855
7856	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7857	APICUpdatePendingInterrupts(pVCpu);
7858
7859	/*
7860	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7861	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7862	*/
7863	/** @todo SMI. SMIs take priority over NMIs. */
7864	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7865	{
7866	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7867	if ( !pVCpu->hm.s.Event.fPending
7868	&& !fBlockNmi
7869	&& !fBlockSti
7870	&& !fBlockMovSS)
7871	{
7872	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7873	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7874	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7875
7876	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7877	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7878	}
7879	else
7880	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7881	}
7882	/*
7883	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7884	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7885	*/
7886	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7887	&& !pVCpu->hm.s.fSingleInstruction)
7888	{
7889	Assert(!DBGFIsStepping(pVCpu));
7890	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7891	AssertRC(rc);
7892	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7893	if ( !pVCpu->hm.s.Event.fPending
7894	&& !fBlockInt
7895	&& !fBlockSti
7896	&& !fBlockMovSS)
7897	{
7898	uint8_t u8Interrupt;
7899	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7900	if (RT_SUCCESS(rc))
7901	{
7902	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7903	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7904	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7905
7906	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7907	}
7908	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7909	{
7910	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7911	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7912	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7913
7914	/*
7915	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7916	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7917	* need to re-set this force-flag here.
7918	*/
7919	}
7920	else
7921	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7922	}
7923	else
7924	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7925	}
7926
7927	return uIntrState;
7928	}
7929
7930
7931	/**
7932	* Sets a pending-debug exception to be delivered to the guest if the guest is
7933	* single-stepping in the VMCS.
7934	*
7935	* @param pVCpu The cross context virtual CPU structure.
7936	*/
7937	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7938	{
7939	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7940	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7941	AssertRC(rc);
7942	}
7943
7944
7945	/**
7946	* Injects any pending events into the guest if the guest is in a state to
7947	* receive them.
7948	*
7949	* @returns Strict VBox status code (i.e. informational status codes too).
7950	* @param pVCpu The cross context virtual CPU structure.
7951	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7952	* out-of-sync. Make sure to update the required fields
7953	* before using them.
7954	* @param uIntrState The VT-x guest-interruptibility state.
7955	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7956	* return VINF_EM_DBG_STEPPED if the event was
7957	* dispatched directly.
7958	*/
7959	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7960	{
7961	HMVMX_ASSERT_PREEMPT_SAFE();
7962	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7963
7964	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7965	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7966
7967	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7968	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7969	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7970	Assert(!TRPMHasTrap(pVCpu));
7971
7972	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7973	if (pVCpu->hm.s.Event.fPending)
7974	{
7975	/*
7976	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7977	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7978	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7979	*
7980	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7981	*/
7982	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7983	#ifdef VBOX_STRICT
7984	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7985	{
7986	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7987	Assert(!fBlockInt);
7988	Assert(!fBlockSti);
7989	Assert(!fBlockMovSS);
7990	}
7991	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7992	{
7993	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7994	Assert(!fBlockSti);
7995	Assert(!fBlockMovSS);
7996	Assert(!fBlockNmi);
7997	}
7998	#endif
7999	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8000	(uint8_t)uIntType));
8001	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
8002	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
8003	fStepping, &uIntrState);
8004	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8005
8006	/* Update the interruptibility-state as it could have been changed by
8007	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
8008	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
8009	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
8010
8011	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8012	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8013	else
8014	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8015	}
8016
8017	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
8018	if ( fBlockSti
8019	\|\| fBlockMovSS)
8020	{
8021	if (!pVCpu->hm.s.fSingleInstruction)
8022	{
8023	/*
8024	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
8025	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
8026	* See Intel spec. 27.3.4 "Saving Non-Register State".
8027	*/
8028	Assert(!DBGFIsStepping(pVCpu));
8029	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8030	AssertRCReturn(rc2, rc2);
8031	if (pMixedCtx->eflags.Bits.u1TF)
8032	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8033	}
8034	else if (pMixedCtx->eflags.Bits.u1TF)
8035	{
8036	/*
8037	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8038	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8039	*/
8040	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8041	uIntrState = 0;
8042	}
8043	}
8044
8045	/*
8046	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8047	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8048	*/
8049	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8050	AssertRC(rc2);
8051
8052	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8053	NOREF(fBlockMovSS); NOREF(fBlockSti);
8054	return rcStrict;
8055	}
8056
8057
8058	/**
8059	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8060	*
8061	* @param pVCpu The cross context virtual CPU structure.
8062	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8063	* out-of-sync. Make sure to update the required fields
8064	* before using them.
8065	*/
8066	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8067	{
8068	NOREF(pMixedCtx);
8069	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8070	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8071	}
8072
8073
8074	/**
8075	* Injects a double-fault (\#DF) exception into the VM.
8076	*
8077	* @returns Strict VBox status code (i.e. informational status codes too).
8078	* @param pVCpu The cross context virtual CPU structure.
8079	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8080	* out-of-sync. Make sure to update the required fields
8081	* before using them.
8082	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8083	* and should return VINF_EM_DBG_STEPPED if the event
8084	* is injected directly (register modified by us, not
8085	* by hardware on VM-entry).
8086	* @param puIntrState Pointer to the current guest interruptibility-state.
8087	* This interruptibility-state will be updated if
8088	* necessary. This cannot not be NULL.
8089	*/
8090	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8091	{
8092	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8093	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8094	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8095	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8096	fStepping, puIntrState);
8097	}
8098
8099
8100	/**
8101	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8102	*
8103	* @param pVCpu The cross context virtual CPU structure.
8104	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8105	* out-of-sync. Make sure to update the required fields
8106	* before using them.
8107	*/
8108	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8109	{
8110	NOREF(pMixedCtx);
8111	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8112	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8113	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8114	}
8115
8116
8117	/**
8118	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8119	*
8120	* @param pVCpu The cross context virtual CPU structure.
8121	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8122	* out-of-sync. Make sure to update the required fields
8123	* before using them.
8124	* @param cbInstr The value of RIP that is to be pushed on the guest
8125	* stack.
8126	*/
8127	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8128	{
8129	NOREF(pMixedCtx);
8130	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8131	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8132	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8133	}
8134
8135
8136	/**
8137	* Injects a general-protection (\#GP) fault into the VM.
8138	*
8139	* @returns Strict VBox status code (i.e. informational status codes too).
8140	* @param pVCpu The cross context virtual CPU structure.
8141	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8142	* out-of-sync. Make sure to update the required fields
8143	* before using them.
8144	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8145	* mode, i.e. in real-mode it's not valid).
8146	* @param u32ErrorCode The error code associated with the \#GP.
8147	* @param fStepping Whether we're running in
8148	* hmR0VmxRunGuestCodeStep() and should return
8149	* VINF_EM_DBG_STEPPED if the event is injected
8150	* directly (register modified by us, not by
8151	* hardware on VM-entry).
8152	* @param puIntrState Pointer to the current guest interruptibility-state.
8153	* This interruptibility-state will be updated if
8154	* necessary. This cannot not be NULL.
8155	*/
8156	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8157	bool fStepping, uint32_t *puIntrState)
8158	{
8159	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8160	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8161	if (fErrorCodeValid)
8162	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8163	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8164	fStepping, puIntrState);
8165	}
8166
8167
8168	#if 0 /* unused */
8169	/**
8170	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8171	* VM.
8172	*
8173	* @param pVCpu The cross context virtual CPU structure.
8174	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8175	* out-of-sync. Make sure to update the required fields
8176	* before using them.
8177	* @param u32ErrorCode The error code associated with the \#GP.
8178	*/
8179	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8180	{
8181	NOREF(pMixedCtx);
8182	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8183	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8184	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8185	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8186	}
8187	#endif /* unused */
8188
8189
8190	/**
8191	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8192	*
8193	* @param pVCpu The cross context virtual CPU structure.
8194	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8195	* out-of-sync. Make sure to update the required fields
8196	* before using them.
8197	* @param uVector The software interrupt vector number.
8198	* @param cbInstr The value of RIP that is to be pushed on the guest
8199	* stack.
8200	*/
8201	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8202	{
8203	NOREF(pMixedCtx);
8204	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8205	if ( uVector == X86_XCPT_BP
8206	\|\| uVector == X86_XCPT_OF)
8207	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8208	else
8209	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8210	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8211	}
8212
8213
8214	/**
8215	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8216	* stack.
8217	*
8218	* @returns Strict VBox status code (i.e. informational status codes too).
8219	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8220	* @param pVM The cross context VM structure.
8221	* @param pMixedCtx Pointer to the guest-CPU context.
8222	* @param uValue The value to push to the guest stack.
8223	*/
8224	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8225	{
8226	/*
8227	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8228	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8229	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8230	*/
8231	if (pMixedCtx->sp == 1)
8232	return VINF_EM_RESET;
8233	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8234	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8235	AssertRC(rc);
8236	return rc;
8237	}
8238
8239
8240	/**
8241	* Injects an event into the guest upon VM-entry by updating the relevant fields
8242	* in the VM-entry area in the VMCS.
8243	*
8244	* @returns Strict VBox status code (i.e. informational status codes too).
8245	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8246	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8247	*
8248	* @param pVCpu The cross context virtual CPU structure.
8249	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8250	* be out-of-sync. Make sure to update the required
8251	* fields before using them.
8252	* @param u64IntInfo The VM-entry interruption-information field.
8253	* @param cbInstr The VM-entry instruction length in bytes (for
8254	* software interrupts, exceptions and privileged
8255	* software exceptions).
8256	* @param u32ErrCode The VM-entry exception error code.
8257	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8258	* @param puIntrState Pointer to the current guest interruptibility-state.
8259	* This interruptibility-state will be updated if
8260	* necessary. This cannot not be NULL.
8261	* @param fStepping Whether we're running in
8262	* hmR0VmxRunGuestCodeStep() and should return
8263	* VINF_EM_DBG_STEPPED if the event is injected
8264	* directly (register modified by us, not by
8265	* hardware on VM-entry).
8266	*
8267	* @remarks Requires CR0!
8268	*/
8269	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8270	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8271	uint32_t *puIntrState)
8272	{
8273	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8274	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8275	Assert(puIntrState);
8276	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8277
8278	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8279	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8280
8281	#ifdef VBOX_STRICT
8282	/*
8283	* Validate the error-code-valid bit for hardware exceptions.
8284	* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8285	*/
8286	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8287	&& !CPUMIsGuestInRealModeEx(pMixedCtx))
8288	{
8289	switch (uVector)
8290	{
8291	case X86_XCPT_PF:
8292	case X86_XCPT_DF:
8293	case X86_XCPT_TS:
8294	case X86_XCPT_NP:
8295	case X86_XCPT_SS:
8296	case X86_XCPT_GP:
8297	case X86_XCPT_AC:
8298	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8299	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8300	RT_FALL_THRU();
8301	default:
8302	break;
8303	}
8304	}
8305	#endif
8306
8307	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8308	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8309	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8310
8311	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8312
8313	/* We require CR0 to check if the guest is in real-mode. */
8314	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8315	AssertRCReturn(rc, rc);
8316
8317	/*
8318	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8319	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8320	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8321	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8322	*/
8323	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8324	{
8325	PVM pVM = pVCpu->CTX_SUFF(pVM);
8326	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8327	{
8328	Assert(PDMVmmDevHeapIsEnabled(pVM));
8329	Assert(pVM->hm.s.vmx.pRealModeTSS);
8330
8331	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8332	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8333	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8334	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8335	AssertRCReturn(rc, rc);
8336	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8337
8338	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8339	size_t const cbIdtEntry = sizeof(X86IDTR16);
8340	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8341	{
8342	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8343	if (uVector == X86_XCPT_DF)
8344	return VINF_EM_RESET;
8345
8346	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8347	if (uVector == X86_XCPT_GP)
8348	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8349
8350	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8351	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8352	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8353	fStepping, puIntrState);
8354	}
8355
8356	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8357	uint16_t uGuestIp = pMixedCtx->ip;
8358	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8359	{
8360	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8361	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8362	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8363	}
8364	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8365	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8366
8367	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8368	X86IDTR16 IdtEntry;
8369	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8370	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8371	AssertRCReturn(rc, rc);
8372
8373	/* Construct the stack frame for the interrupt/exception handler. */
8374	VBOXSTRICTRC rcStrict;
8375	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8376	if (rcStrict == VINF_SUCCESS)
8377	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8378	if (rcStrict == VINF_SUCCESS)
8379	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8380
8381	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8382	if (rcStrict == VINF_SUCCESS)
8383	{
8384	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8385	pMixedCtx->rip = IdtEntry.offSel;
8386	pMixedCtx->cs.Sel = IdtEntry.uSel;
8387	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8388	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8389	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8390	&& uVector == X86_XCPT_PF)
8391	pMixedCtx->cr2 = GCPtrFaultAddress;
8392
8393	/* If any other guest-state bits are changed here, make sure to update
8394	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8395	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8396	\| HM_CHANGED_GUEST_RIP
8397	\| HM_CHANGED_GUEST_RFLAGS
8398	\| HM_CHANGED_GUEST_RSP);
8399
8400	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8401	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8402	{
8403	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8404	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8405	Log4(("Clearing inhibition due to STI.\n"));
8406	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8407	}
8408	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8409	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8410
8411	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8412	it, if we are returning to ring-3 before executing guest code. */
8413	pVCpu->hm.s.Event.fPending = false;
8414
8415	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8416	if (fStepping)
8417	rcStrict = VINF_EM_DBG_STEPPED;
8418	}
8419	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8420	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8421	return rcStrict;
8422	}
8423
8424	/*
8425	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8426	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8427	*/
8428	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8429	}
8430
8431	/* Validate. */
8432	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8433	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8434	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8435
8436	/* Inject. */
8437	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8438	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8439	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8440	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8441
8442	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8443	&& uVector == X86_XCPT_PF)
8444	pMixedCtx->cr2 = GCPtrFaultAddress;
8445
8446	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8447	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8448
8449	AssertRCReturn(rc, rc);
8450	return VINF_SUCCESS;
8451	}
8452
8453
8454	/**
8455	* Clears the interrupt-window exiting control in the VMCS and if necessary
8456	* clears the current event in the VMCS as well.
8457	*
8458	* @returns VBox status code.
8459	* @param pVCpu The cross context virtual CPU structure.
8460	*
8461	* @remarks Use this function only to clear events that have not yet been
8462	* delivered to the guest but are injected in the VMCS!
8463	* @remarks No-long-jump zone!!!
8464	*/
8465	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8466	{
8467	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8468
8469	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8470	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8471
8472	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8473	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8474	}
8475
8476
8477	/**
8478	* Enters the VT-x session.
8479	*
8480	* @returns VBox status code.
8481	* @param pVM The cross context VM structure.
8482	* @param pVCpu The cross context virtual CPU structure.
8483	* @param pCpu Pointer to the CPU info struct.
8484	*/
8485	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8486	{
8487	AssertPtr(pVM);
8488	AssertPtr(pVCpu);
8489	Assert(pVM->hm.s.vmx.fSupported);
8490	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8491	NOREF(pCpu); NOREF(pVM);
8492
8493	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8494	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8495
8496	#ifdef VBOX_STRICT
8497	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8498	RTCCUINTREG uHostCR4 = ASMGetCR4();
8499	if (!(uHostCR4 & X86_CR4_VMXE))
8500	{
8501	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8502	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8503	}
8504	#endif
8505
8506	/*
8507	* Load the VCPU's VMCS as the current (and active) one.
8508	*/
8509	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8510	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8511	if (RT_FAILURE(rc))
8512	return rc;
8513
8514	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8515	pVCpu->hm.s.fLeaveDone = false;
8516	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8517
8518	return VINF_SUCCESS;
8519	}
8520
8521
8522	/**
8523	* The thread-context callback (only on platforms which support it).
8524	*
8525	* @param enmEvent The thread-context event.
8526	* @param pVCpu The cross context virtual CPU structure.
8527	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8528	* @thread EMT(pVCpu)
8529	*/
8530	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8531	{
8532	NOREF(fGlobalInit);
8533
8534	switch (enmEvent)
8535	{
8536	case RTTHREADCTXEVENT_OUT:
8537	{
8538	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8539	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8540	VMCPU_ASSERT_EMT(pVCpu);
8541
8542	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8543
8544	/* No longjmps (logger flushes, locks) in this fragile context. */
8545	VMMRZCallRing3Disable(pVCpu);
8546	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8547
8548	/*
8549	* Restore host-state (FPU, debug etc.)
8550	*/
8551	if (!pVCpu->hm.s.fLeaveDone)
8552	{
8553	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8554	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8555	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8556	pVCpu->hm.s.fLeaveDone = true;
8557	}
8558
8559	/* Leave HM context, takes care of local init (term). */
8560	int rc = HMR0LeaveCpu(pVCpu);
8561	AssertRC(rc); NOREF(rc);
8562
8563	/* Restore longjmp state. */
8564	VMMRZCallRing3Enable(pVCpu);
8565	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8566	break;
8567	}
8568
8569	case RTTHREADCTXEVENT_IN:
8570	{
8571	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8572	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8573	VMCPU_ASSERT_EMT(pVCpu);
8574
8575	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8576	VMMRZCallRing3Disable(pVCpu);
8577	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8578
8579	/* Initialize the bare minimum state required for HM. This takes care of
8580	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8581	int rc = HMR0EnterCpu(pVCpu);
8582	AssertRC(rc);
8583	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8584
8585	/* Load the active VMCS as the current one. */
8586	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8587	{
8588	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8589	AssertRC(rc); NOREF(rc);
8590	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8591	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8592	}
8593	pVCpu->hm.s.fLeaveDone = false;
8594
8595	/* Restore longjmp state. */
8596	VMMRZCallRing3Enable(pVCpu);
8597	break;
8598	}
8599
8600	default:
8601	break;
8602	}
8603	}
8604
8605
8606	/**
8607	* Saves the host state in the VMCS host-state.
8608	* Sets up the VM-exit MSR-load area.
8609	*
8610	* The CPU state will be loaded from these fields on every successful VM-exit.
8611	*
8612	* @returns VBox status code.
8613	* @param pVM The cross context VM structure.
8614	* @param pVCpu The cross context virtual CPU structure.
8615	*
8616	* @remarks No-long-jump zone!!!
8617	*/
8618	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8619	{
8620	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8621
8622	int rc = VINF_SUCCESS;
8623	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8624	{
8625	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8626	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8627
8628	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8629	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8630
8631	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8632	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8633
8634	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8635	}
8636	return rc;
8637	}
8638
8639
8640	/**
8641	* Saves the host state in the VMCS host-state.
8642	*
8643	* @returns VBox status code.
8644	* @param pVM The cross context VM structure.
8645	* @param pVCpu The cross context virtual CPU structure.
8646	*
8647	* @remarks No-long-jump zone!!!
8648	*/
8649	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8650	{
8651	AssertPtr(pVM);
8652	AssertPtr(pVCpu);
8653
8654	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8655
8656	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8657	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8658	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8659	return hmR0VmxSaveHostState(pVM, pVCpu);
8660	}
8661
8662
8663	/**
8664	* Loads the guest state into the VMCS guest-state area.
8665	*
8666	* The will typically be done before VM-entry when the guest-CPU state and the
8667	* VMCS state may potentially be out of sync.
8668	*
8669	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8670	* VM-entry controls.
8671	* Sets up the appropriate VMX non-root function to execute guest code based on
8672	* the guest CPU mode.
8673	*
8674	* @returns VBox strict status code.
8675	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8676	* without unrestricted guest access and the VMMDev is not presently
8677	* mapped (e.g. EFI32).
8678	*
8679	* @param pVM The cross context VM structure.
8680	* @param pVCpu The cross context virtual CPU structure.
8681	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8682	* out-of-sync. Make sure to update the required fields
8683	* before using them.
8684	*
8685	* @remarks No-long-jump zone!!!
8686	*/
8687	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8688	{
8689	AssertPtr(pVM);
8690	AssertPtr(pVCpu);
8691	AssertPtr(pMixedCtx);
8692	HMVMX_ASSERT_PREEMPT_SAFE();
8693
8694	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8695
8696	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8697
8698	/* Determine real-on-v86 mode. */
8699	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8700	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8701	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8702	{
8703	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8704	}
8705
8706	/*
8707	* Load the guest-state into the VMCS.
8708	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8709	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8710	*/
8711	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8712	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8713
8714	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8715	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8716	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8717
8718	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8719	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8720	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8721
8722	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8723	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8724
8725	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8726	if (rcStrict == VINF_SUCCESS)
8727	{ /* likely */ }
8728	else
8729	{
8730	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8731	return rcStrict;
8732	}
8733
8734	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8735	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8736	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8737
8738	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8739	determine we don't have to swap EFER after all. */
8740	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8741	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8742
8743	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8744	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8745
8746	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8747	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8748
8749	/*
8750	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8751	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8752	*/
8753	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8754	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8755
8756	/* Clear any unused and reserved bits. */
8757	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2
8758	\| HM_CHANGED_GUEST_HWVIRT);
8759
8760	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8761	return rc;
8762	}
8763
8764
8765	/**
8766	* Loads the state shared between the host and guest into the VMCS.
8767	*
8768	* @param pVM The cross context VM structure.
8769	* @param pVCpu The cross context virtual CPU structure.
8770	* @param pCtx Pointer to the guest-CPU context.
8771	*
8772	* @remarks No-long-jump zone!!!
8773	*/
8774	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8775	{
8776	NOREF(pVM);
8777
8778	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8779	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8780
8781	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8782	{
8783	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8784	AssertRC(rc);
8785	}
8786
8787	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8788	{
8789	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8790	AssertRC(rc);
8791
8792	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8793	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8794	{
8795	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8796	AssertRC(rc);
8797	}
8798	}
8799
8800	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS))
8801	{
8802	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8803	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
8804	}
8805
8806	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8807	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
8808	{
8809	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8810	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8811	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8812	AssertRC(rc);
8813	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
8814	}
8815
8816	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8817	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8818	}
8819
8820
8821	/**
8822	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8823	*
8824	* @returns Strict VBox status code (i.e. informational status codes too).
8825	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8826	* without unrestricted guest access and the VMMDev is not presently
8827	* mapped (e.g. EFI32).
8828	*
8829	* @param pVM The cross context VM structure.
8830	* @param pVCpu The cross context virtual CPU structure.
8831	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8832	* out-of-sync. Make sure to update the required fields
8833	* before using them.
8834	*
8835	* @remarks No-long-jump zone!!!
8836	*/
8837	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8838	{
8839	HMVMX_ASSERT_PREEMPT_SAFE();
8840	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8841	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8842
8843	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8844	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8845	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8846	#endif
8847
8848	/*
8849	* RIP is what changes the most often and hence if it's the only bit needing to be
8850	* updated, we shall handle it early for performance reasons.
8851	*/
8852	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8853	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8854	{
8855	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8856	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8857	{ /* likely */}
8858	else
8859	{
8860	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8861	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8862	}
8863	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8864	}
8865	else if (HMCPU_CF_VALUE(pVCpu))
8866	{
8867	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8868	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8869	{ /* likely */}
8870	else
8871	{
8872	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8873	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8874	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8875	return rcStrict;
8876	}
8877	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8878	}
8879
8880	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8881	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8882	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8883	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8884	return rcStrict;
8885	}
8886
8887
8888	/**
8889	* Does the preparations before executing guest code in VT-x.
8890	*
8891	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8892	* recompiler/IEM. We must be cautious what we do here regarding committing
8893	* guest-state information into the VMCS assuming we assuredly execute the
8894	* guest in VT-x mode.
8895	*
8896	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8897	* the common-state (TRPM/forceflags), we must undo those changes so that the
8898	* recompiler/IEM can (and should) use them when it resumes guest execution.
8899	* Otherwise such operations must be done when we can no longer exit to ring-3.
8900	*
8901	* @returns Strict VBox status code (i.e. informational status codes too).
8902	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8903	* have been disabled.
8904	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8905	* double-fault into the guest.
8906	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8907	* dispatched directly.
8908	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8909	*
8910	* @param pVM The cross context VM structure.
8911	* @param pVCpu The cross context virtual CPU structure.
8912	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8913	* out-of-sync. Make sure to update the required fields
8914	* before using them.
8915	* @param pVmxTransient Pointer to the VMX transient structure.
8916	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8917	* us ignore some of the reasons for returning to
8918	* ring-3, and return VINF_EM_DBG_STEPPED if event
8919	* dispatching took place.
8920	*/
8921	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8922	{
8923	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8924
8925	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8926	PGMRZDynMapFlushAutoSet(pVCpu);
8927	#endif
8928
8929	/* Check force flag actions that might require us to go back to ring-3. */
8930	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8931	if (rcStrict == VINF_SUCCESS)
8932	{ /* FFs doesn't get set all the time. */ }
8933	else
8934	return rcStrict;
8935
8936	#ifndef IEM_VERIFICATION_MODE_FULL
8937	/*
8938	* Setup the virtualized-APIC accesses.
8939	*
8940	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8941	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8942	*
8943	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
8944	*/
8945	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8946	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8947	&& PDMHasApic(pVM))
8948	{
8949	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8950	Assert(u64MsrApicBase);
8951	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8952
8953	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8954
8955	/* Unalias any existing mapping. */
8956	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8957	AssertRCReturn(rc, rc);
8958
8959	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8960	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8961	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8962	AssertRCReturn(rc, rc);
8963
8964	/* Update the per-VCPU cache of the APIC base MSR. */
8965	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8966	}
8967	#endif /* !IEM_VERIFICATION_MODE_FULL */
8968
8969	if (TRPMHasTrap(pVCpu))
8970	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8971	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8972
8973	/*
8974	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8975	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8976	*/
8977	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8978	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8979	{ /* likely */ }
8980	else
8981	{
8982	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8983	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8984	return rcStrict;
8985	}
8986
8987	/*
8988	* No longjmps to ring-3 from this point on!!!
8989	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8990	* This also disables flushing of the R0-logger instance (if any).
8991	*/
8992	VMMRZCallRing3Disable(pVCpu);
8993
8994	/*
8995	* Load the guest state bits.
8996	*
8997	* We cannot perform longjmps while loading the guest state because we do not preserve the
8998	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8999	* CPU migration.
9000	*
9001	* If we are injecting events to a real-on-v86 mode guest, we will have to update
9002	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
9003	* Hence, loading of the guest state needs to be done -after- injection of events.
9004	*/
9005	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
9006	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9007	{ /* likely */ }
9008	else
9009	{
9010	VMMRZCallRing3Enable(pVCpu);
9011	return rcStrict;
9012	}
9013
9014	/*
9015	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
9016	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
9017	*
9018	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
9019	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
9020	*
9021	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
9022	* executing guest code.
9023	*/
9024	pVmxTransient->fEFlags = ASMIntDisableFlags();
9025
9026	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
9027	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
9028	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
9029	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
9030	{
9031	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
9032	{
9033	pVCpu->hm.s.Event.fPending = false;
9034
9035	/*
9036	* We've injected any pending events. This is really the point of no return (to ring-3).
9037	*
9038	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
9039	* returns from this function, so don't enable them here.
9040	*/
9041	return VINF_SUCCESS;
9042	}
9043
9044	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9045	rcStrict = VINF_EM_RAW_INTERRUPT;
9046	}
9047	else
9048	{
9049	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9050	rcStrict = VINF_EM_RAW_TO_R3;
9051	}
9052
9053	ASMSetFlags(pVmxTransient->fEFlags);
9054	VMMRZCallRing3Enable(pVCpu);
9055
9056	return rcStrict;
9057	}
9058
9059
9060	/**
9061	* Prepares to run guest code in VT-x and we've committed to doing so. This
9062	* means there is no backing out to ring-3 or anywhere else at this
9063	* point.
9064	*
9065	* @param pVM The cross context VM structure.
9066	* @param pVCpu The cross context virtual CPU structure.
9067	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9068	* out-of-sync. Make sure to update the required fields
9069	* before using them.
9070	* @param pVmxTransient Pointer to the VMX transient structure.
9071	*
9072	* @remarks Called with preemption disabled.
9073	* @remarks No-long-jump zone!!!
9074	*/
9075	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9076	{
9077	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9078	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9079	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9080
9081	/*
9082	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9083	*/
9084	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9085	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9086
9087	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9088	if (!CPUMIsGuestFPUStateActive(pVCpu))
9089	{
9090	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9091	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9092	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9093	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9094	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9095	}
9096	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9097	#endif
9098
9099	if ( pVCpu->hm.s.fPreloadGuestFpu
9100	&& !CPUMIsGuestFPUStateActive(pVCpu))
9101	{
9102	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9103	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9104	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9105	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9106	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9107	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9108	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9109	}
9110
9111	/*
9112	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9113	*/
9114	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9115	&& pVCpu->hm.s.vmx.cMsrs > 0)
9116	{
9117	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9118	}
9119
9120	/*
9121	* Load the host state bits as we may've been preempted (only happens when
9122	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9123	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9124	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9125	* See @bugref{8432}.
9126	*/
9127	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9128	{
9129	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9130	AssertRC(rc);
9131	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9132	}
9133	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9134
9135	/*
9136	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9137	*/
9138	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9139	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9140	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9141
9142	/* Store status of the shared guest-host state at the time of VM-entry. */
9143	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9144	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9145	{
9146	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9147	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9148	}
9149	else
9150	#endif
9151	{
9152	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9153	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9154	}
9155	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9156
9157	/*
9158	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9159	*/
9160	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9161	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9162
9163	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9164	RTCPUID idCurrentCpu = pCpu->idCpu;
9165	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9166	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9167	{
9168	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9169	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9170	}
9171
9172	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9173	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9174	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9175	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9176
9177	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9178
9179	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9180	to start executing. */
9181
9182	/*
9183	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9184	*/
9185	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9186	{
9187	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9188	{
9189	bool fMsrUpdated;
9190	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9191	AssertRC(rc2);
9192	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9193
9194	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9195	&fMsrUpdated);
9196	AssertRC(rc2);
9197	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9198
9199	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9200	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9201	}
9202	else
9203	{
9204	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9205	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9206	}
9207	}
9208
9209	if (pVM->cpum.ro.GuestFeatures.fIbrs)
9210	{
9211	bool fMsrUpdated;
9212	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9213	AssertRC(rc2);
9214	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9215
9216	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMR0GetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
9217	&fMsrUpdated);
9218	AssertRC(rc2);
9219	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9220	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9221	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9222	}
9223
9224	#ifdef VBOX_STRICT
9225	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9226	hmR0VmxCheckHostEferMsr(pVCpu);
9227	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9228	#endif
9229	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9230	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9231	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9232	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9233	#endif
9234	}
9235
9236
9237	/**
9238	* Performs some essential restoration of state after running guest code in
9239	* VT-x.
9240	*
9241	* @param pVM The cross context VM structure.
9242	* @param pVCpu The cross context virtual CPU structure.
9243	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9244	* out-of-sync. Make sure to update the required fields
9245	* before using them.
9246	* @param pVmxTransient Pointer to the VMX transient structure.
9247	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9248	*
9249	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9250	*
9251	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9252	* unconditionally when it is safe to do so.
9253	*/
9254	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9255	{
9256	NOREF(pVM);
9257
9258	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9259
9260	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9261	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9262	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9263	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9264	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9265	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9266
9267	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9268	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9269
9270	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9271	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9272	Assert(!ASMIntAreEnabled());
9273	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9274
9275	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9276	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9277	{
9278	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9279	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9280	}
9281	#endif
9282
9283	#if HC_ARCH_BITS == 64
9284	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9285	#endif
9286	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9287	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9288	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9289	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9290	#else
9291	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9292	#endif
9293	#ifdef VBOX_STRICT
9294	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9295	#endif
9296	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9297
9298	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9299	uint32_t uExitReason;
9300	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9301	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9302	AssertRC(rc);
9303	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9304	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9305
9306	if (rcVMRun == VINF_SUCCESS)
9307	{
9308	/*
9309	* Update the VM-exit history array here even if the VM-entry failed due to:
9310	* - Invalid guest state.
9311	* - MSR loading.
9312	* - Machine-check event.
9313	*
9314	* In any of the above cases we will still have a "valid" VM-exit reason
9315	* despite @a fVMEntryFailed being false.
9316	*
9317	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9318	*/
9319	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9320
9321	if (!pVmxTransient->fVMEntryFailed)
9322	{
9323	/** @todo We can optimize this by only syncing with our force-flags when
9324	* really needed and keeping the VMCS state as it is for most
9325	* VM-exits. */
9326	/* Update the guest interruptibility-state from the VMCS. */
9327	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9328
9329	/*
9330	* Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9331	* as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9332	* on the longjmp path to ring-3 while saving the (rest of) the guest state,
9333	* see @bugref{6208#c63}.
9334	*/
9335	VMMRZCallRing3Enable(pVCpu);
9336
9337	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9338	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9339	AssertRC(rc);
9340	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9341	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9342	AssertRC(rc);
9343	#endif
9344
9345	/*
9346	* Sync the TPR shadow with our APIC state.
9347	*/
9348	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9349	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9350	{
9351	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9352	AssertRC(rc);
9353	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
9354	}
9355
9356	return;
9357	}
9358	}
9359	else
9360	{
9361	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9362	pVmxTransient->fVMEntryFailed));
9363	}
9364
9365	VMMRZCallRing3Enable(pVCpu);
9366	}
9367
9368
9369	/**
9370	* Runs the guest code using VT-x the normal way.
9371	*
9372	* @returns VBox status code.
9373	* @param pVM The cross context VM structure.
9374	* @param pVCpu The cross context virtual CPU structure.
9375	* @param pCtx Pointer to the guest-CPU context.
9376	*
9377	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9378	*/
9379	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9380	{
9381	VMXTRANSIENT VmxTransient;
9382	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9383	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9384	uint32_t cLoops = 0;
9385
9386	for (;; cLoops++)
9387	{
9388	Assert(!HMR0SuspendPending());
9389	HMVMX_ASSERT_CPU_SAFE();
9390
9391	/* Preparatory work for running guest code, this may force us to return
9392	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9393	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9394	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9395	if (rcStrict != VINF_SUCCESS)
9396	break;
9397
9398	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9399	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9400	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9401
9402	/* Restore any residual host-state and save any bits shared between host
9403	and guest into the guest-CPU state. Re-enables interrupts! */
9404	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9405
9406	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9407	if (RT_SUCCESS(rcRun))
9408	{ /* very likely */ }
9409	else
9410	{
9411	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9412	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9413	return rcRun;
9414	}
9415
9416	/* Profile the VM-exit. */
9417	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9418	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9419	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9420	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9421	HMVMX_START_EXIT_DISPATCH_PROF();
9422
9423	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9424
9425	/* Handle the VM-exit. */
9426	#ifdef HMVMX_USE_FUNCTION_TABLE
9427	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9428	#else
9429	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9430	#endif
9431	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9432	if (rcStrict == VINF_SUCCESS)
9433	{
9434	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9435	continue; /* likely */
9436	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9437	rcStrict = VINF_EM_RAW_INTERRUPT;
9438	}
9439	break;
9440	}
9441
9442	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9443	return rcStrict;
9444	}
9445
9446
9447
9448	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9449	* probes.
9450	*
9451	* The following few functions and associated structure contains the bloat
9452	* necessary for providing detailed debug events and dtrace probes as well as
9453	* reliable host side single stepping. This works on the principle of
9454	* "subclassing" the normal execution loop and workers. We replace the loop
9455	* method completely and override selected helpers to add necessary adjustments
9456	* to their core operation.
9457	*
9458	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9459	* any performance for debug and analysis features.
9460	*
9461	* @{
9462	*/
9463
9464	/**
9465	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9466	* the debug run loop.
9467	*/
9468	typedef struct VMXRUNDBGSTATE
9469	{
9470	/** The RIP we started executing at. This is for detecting that we stepped. */
9471	uint64_t uRipStart;
9472	/** The CS we started executing with. */
9473	uint16_t uCsStart;
9474
9475	/** Whether we've actually modified the 1st execution control field. */
9476	bool fModifiedProcCtls : 1;
9477	/** Whether we've actually modified the 2nd execution control field. */
9478	bool fModifiedProcCtls2 : 1;
9479	/** Whether we've actually modified the exception bitmap. */
9480	bool fModifiedXcptBitmap : 1;
9481
9482	/** We desire the modified the CR0 mask to be cleared. */
9483	bool fClearCr0Mask : 1;
9484	/** We desire the modified the CR4 mask to be cleared. */
9485	bool fClearCr4Mask : 1;
9486	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9487	uint32_t fCpe1Extra;
9488	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9489	uint32_t fCpe1Unwanted;
9490	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9491	uint32_t fCpe2Extra;
9492	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9493	uint32_t bmXcptExtra;
9494	/** The sequence number of the Dtrace provider settings the state was
9495	* configured against. */
9496	uint32_t uDtraceSettingsSeqNo;
9497	/** VM-exits to check (one bit per VM-exit). */
9498	uint32_t bmExitsToCheck[3];
9499
9500	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9501	uint32_t fProcCtlsInitial;
9502	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9503	uint32_t fProcCtls2Initial;
9504	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9505	uint32_t bmXcptInitial;
9506	} VMXRUNDBGSTATE;
9507	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9508	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9509
9510
9511	/**
9512	* Initializes the VMXRUNDBGSTATE structure.
9513	*
9514	* @param pVCpu The cross context virtual CPU structure of the
9515	* calling EMT.
9516	* @param pCtx The CPU register context to go with @a pVCpu.
9517	* @param pDbgState The structure to initialize.
9518	*/
9519	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9520	{
9521	pDbgState->uRipStart = pCtx->rip;
9522	pDbgState->uCsStart = pCtx->cs.Sel;
9523
9524	pDbgState->fModifiedProcCtls = false;
9525	pDbgState->fModifiedProcCtls2 = false;
9526	pDbgState->fModifiedXcptBitmap = false;
9527	pDbgState->fClearCr0Mask = false;
9528	pDbgState->fClearCr4Mask = false;
9529	pDbgState->fCpe1Extra = 0;
9530	pDbgState->fCpe1Unwanted = 0;
9531	pDbgState->fCpe2Extra = 0;
9532	pDbgState->bmXcptExtra = 0;
9533	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9534	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9535	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9536	}
9537
9538
9539	/**
9540	* Updates the VMSC fields with changes requested by @a pDbgState.
9541	*
9542	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9543	* immediately before executing guest code, i.e. when interrupts are disabled.
9544	* We don't check status codes here as we cannot easily assert or return in the
9545	* latter case.
9546	*
9547	* @param pVCpu The cross context virtual CPU structure.
9548	* @param pDbgState The debug state.
9549	*/
9550	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9551	{
9552	/*
9553	* Ensure desired flags in VMCS control fields are set.
9554	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9555	*
9556	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9557	* there should be no stale data in pCtx at this point.
9558	*/
9559	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9560	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9561	{
9562	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9563	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9564	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9565	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9566	pDbgState->fModifiedProcCtls = true;
9567	}
9568
9569	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9570	{
9571	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9572	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9573	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9574	pDbgState->fModifiedProcCtls2 = true;
9575	}
9576
9577	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9578	{
9579	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9580	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9581	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9582	pDbgState->fModifiedXcptBitmap = true;
9583	}
9584
9585	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9586	{
9587	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9588	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9589	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9590	}
9591
9592	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9593	{
9594	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9595	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9596	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9597	}
9598	}
9599
9600
9601	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9602	{
9603	/*
9604	* Restore VM-exit control settings as we may not reenter this function the
9605	* next time around.
9606	*/
9607	/* We reload the initial value, trigger what we can of recalculations the
9608	next time around. From the looks of things, that's all that's required atm. */
9609	if (pDbgState->fModifiedProcCtls)
9610	{
9611	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9612	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9613	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9614	AssertRCReturn(rc2, rc2);
9615	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9616	}
9617
9618	/* We're currently the only ones messing with this one, so just restore the
9619	cached value and reload the field. */
9620	if ( pDbgState->fModifiedProcCtls2
9621	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9622	{
9623	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9624	AssertRCReturn(rc2, rc2);
9625	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9626	}
9627
9628	/* If we've modified the exception bitmap, we restore it and trigger
9629	reloading and partial recalculation the next time around. */
9630	if (pDbgState->fModifiedXcptBitmap)
9631	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9632
9633	return rcStrict;
9634	}
9635
9636
9637	/**
9638	* Configures VM-exit controls for current DBGF and DTrace settings.
9639	*
9640	* This updates @a pDbgState and the VMCS execution control fields to reflect
9641	* the necessary VM-exits demanded by DBGF and DTrace.
9642	*
9643	* @param pVM The cross context VM structure.
9644	* @param pVCpu The cross context virtual CPU structure.
9645	* @param pCtx Pointer to the guest-CPU context.
9646	* @param pDbgState The debug state.
9647	* @param pVmxTransient Pointer to the VMX transient structure. May update
9648	* fUpdateTscOffsettingAndPreemptTimer.
9649	*/
9650	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9651	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9652	{
9653	/*
9654	* Take down the dtrace serial number so we can spot changes.
9655	*/
9656	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9657	ASMCompilerBarrier();
9658
9659	/*
9660	* We'll rebuild most of the middle block of data members (holding the
9661	* current settings) as we go along here, so start by clearing it all.
9662	*/
9663	pDbgState->bmXcptExtra = 0;
9664	pDbgState->fCpe1Extra = 0;
9665	pDbgState->fCpe1Unwanted = 0;
9666	pDbgState->fCpe2Extra = 0;
9667	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9668	pDbgState->bmExitsToCheck[i] = 0;
9669
9670	/*
9671	* Software interrupts (INT XXh) - no idea how to trigger these...
9672	*/
9673	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9674	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9675	{
9676	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9677	}
9678
9679	/*
9680	* INT3 breakpoints - triggered by #BP exceptions.
9681	*/
9682	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9683	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9684
9685	/*
9686	* Exception bitmap and XCPT events+probes.
9687	*/
9688	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9689	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9690	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9691
9692	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9693	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9694	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9695	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9696	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9697	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9698	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9699	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9700	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9701	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9702	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9703	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9704	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9705	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9706	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9707	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9708	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9709	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9710
9711	if (pDbgState->bmXcptExtra)
9712	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9713
9714	/*
9715	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9716	*
9717	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9718	* So, when adding/changing/removing please don't forget to update it.
9719	*
9720	* Some of the macros are picking up local variables to save horizontal space,
9721	* (being able to see it in a table is the lesser evil here).
9722	*/
9723	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9724	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9725	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9726	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9727	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9728	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9729	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9730	} else do { } while (0)
9731	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9732	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9733	{ \
9734	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9735	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9736	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9737	} else do { } while (0)
9738	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9739	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9740	{ \
9741	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9742	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9743	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9744	} else do { } while (0)
9745	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9746	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9747	{ \
9748	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9749	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9750	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9751	} else do { } while (0)
9752
9753	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9754	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9755	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9756	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9757	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9758
9759	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9761	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9762	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9763	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9764	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9765	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9766	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9767	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9768	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9769	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9770	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9771	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9772	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9773	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9774	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9775	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9776	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9777	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9778	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9779	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9780	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9781	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9782	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9783	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9784	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9785	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9786	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9787	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9788	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9789	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9790	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9791	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9792	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9793	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9794	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9795
9796	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9797	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9798	{
9799	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9800	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9801	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9802	AssertRC(rc2);
9803
9804	#if 0 /** @todo fix me */
9805	pDbgState->fClearCr0Mask = true;
9806	pDbgState->fClearCr4Mask = true;
9807	#endif
9808	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9809	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9810	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9811	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9812	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9813	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9814	require clearing here and in the loop if we start using it. */
9815	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9816	}
9817	else
9818	{
9819	if (pDbgState->fClearCr0Mask)
9820	{
9821	pDbgState->fClearCr0Mask = false;
9822	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9823	}
9824	if (pDbgState->fClearCr4Mask)
9825	{
9826	pDbgState->fClearCr4Mask = false;
9827	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9828	}
9829	}
9830	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9831	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9832
9833	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9834	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9835	{
9836	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9837	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9838	}
9839	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9840	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9841
9842	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9843	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9844	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9845	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9846	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9847	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9848	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9849	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9850	#if 0 /** @todo too slow, fix handler. */
9851	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9852	#endif
9853	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9854
9855	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9856	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9857	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9858	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9859	{
9860	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9861	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9862	}
9863	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9864	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9865	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9866	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9867
9868	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9869	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9870	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9871	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9872	{
9873	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9874	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9875	}
9876	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9877	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9878	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9879	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9880
9881	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9882	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9883	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9884	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9885	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9886	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9887	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9888	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9889	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9890	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9891	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9892	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9893	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9894	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9895	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9896	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9897	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9898	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9899	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9900	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9901	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9902	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9903
9904	#undef IS_EITHER_ENABLED
9905	#undef SET_ONLY_XBM_IF_EITHER_EN
9906	#undef SET_CPE1_XBM_IF_EITHER_EN
9907	#undef SET_CPEU_XBM_IF_EITHER_EN
9908	#undef SET_CPE2_XBM_IF_EITHER_EN
9909
9910	/*
9911	* Sanitize the control stuff.
9912	*/
9913	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9914	if (pDbgState->fCpe2Extra)
9915	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9916	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9917	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9918	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9919	{
9920	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9921	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9922	}
9923
9924	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9925	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9926	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9927	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9928	}
9929
9930
9931	/**
9932	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9933	* appropriate.
9934	*
9935	* The caller has checked the VM-exit against the
9936	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9937	* already, so we don't have to do that either.
9938	*
9939	* @returns Strict VBox status code (i.e. informational status codes too).
9940	* @param pVM The cross context VM structure.
9941	* @param pVCpu The cross context virtual CPU structure.
9942	* @param pMixedCtx Pointer to the guest-CPU context.
9943	* @param pVmxTransient Pointer to the VMX-transient structure.
9944	* @param uExitReason The VM-exit reason.
9945	*
9946	* @remarks The name of this function is displayed by dtrace, so keep it short
9947	* and to the point. No longer than 33 chars long, please.
9948	*/
9949	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9950	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9951	{
9952	/*
9953	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9954	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9955	*
9956	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9957	* does. Must add/change/remove both places. Same ordering, please.
9958	*
9959	* Added/removed events must also be reflected in the next section
9960	* where we dispatch dtrace events.
9961	*/
9962	bool fDtrace1 = false;
9963	bool fDtrace2 = false;
9964	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9965	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9966	uint32_t uEventArg = 0;
9967	#define SET_EXIT(a_EventSubName) \
9968	do { \
9969	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9970	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9971	} while (0)
9972	#define SET_BOTH(a_EventSubName) \
9973	do { \
9974	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9975	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9976	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9977	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9978	} while (0)
9979	switch (uExitReason)
9980	{
9981	case VMX_EXIT_MTF:
9982	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9983
9984	case VMX_EXIT_XCPT_OR_NMI:
9985	{
9986	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9987	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9988	{
9989	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9990	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9991	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9992	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9993	{
9994	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9995	{
9996	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9997	uEventArg = pVmxTransient->uExitIntErrorCode;
9998	}
9999	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
10000	switch (enmEvent1)
10001	{
10002	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
10003	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
10004	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
10005	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
10006	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
10007	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
10008	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
10009	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
10010	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
10011	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
10012	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
10013	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
10014	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
10015	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
10016	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
10017	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
10018	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
10019	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
10020	default: break;
10021	}
10022	}
10023	else
10024	AssertFailed();
10025	break;
10026
10027	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
10028	uEventArg = idxVector;
10029	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
10030	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
10031	break;
10032	}
10033	break;
10034	}
10035
10036	case VMX_EXIT_TRIPLE_FAULT:
10037	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
10038	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
10039	break;
10040	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
10041	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
10042	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
10043	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
10044	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
10045
10046	/* Instruction specific VM-exits: */
10047	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10048	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10049	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10050	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10051	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10052	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10053	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10054	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10055	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10056	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10057	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10058	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10059	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10060	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10061	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10062	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10063	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10064	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10065	case VMX_EXIT_MOV_CRX:
10066	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10067	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10068	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10069	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10070	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10071	SET_BOTH(CRX_READ);
10072	else
10073	SET_BOTH(CRX_WRITE);
10074	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10075	break;
10076	case VMX_EXIT_MOV_DRX:
10077	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10078	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10079	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10080	SET_BOTH(DRX_READ);
10081	else
10082	SET_BOTH(DRX_WRITE);
10083	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10084	break;
10085	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10086	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10087	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10088	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10089	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10090	case VMX_EXIT_XDTR_ACCESS:
10091	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10092	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10093	{
10094	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10095	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10096	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10097	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10098	}
10099	break;
10100
10101	case VMX_EXIT_TR_ACCESS:
10102	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10103	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10104	{
10105	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10106	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10107	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10108	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10109	}
10110	break;
10111
10112	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10113	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10114	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10115	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10116	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10117	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10118	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10119	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10120	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10121	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10122	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10123
10124	/* Events that aren't relevant at this point. */
10125	case VMX_EXIT_EXT_INT:
10126	case VMX_EXIT_INT_WINDOW:
10127	case VMX_EXIT_NMI_WINDOW:
10128	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10129	case VMX_EXIT_PREEMPT_TIMER:
10130	case VMX_EXIT_IO_INSTR:
10131	break;
10132
10133	/* Errors and unexpected events. */
10134	case VMX_EXIT_INIT_SIGNAL:
10135	case VMX_EXIT_SIPI:
10136	case VMX_EXIT_IO_SMI:
10137	case VMX_EXIT_SMI:
10138	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10139	case VMX_EXIT_ERR_MSR_LOAD:
10140	case VMX_EXIT_ERR_MACHINE_CHECK:
10141	break;
10142
10143	default:
10144	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10145	break;
10146	}
10147	#undef SET_BOTH
10148	#undef SET_EXIT
10149
10150	/*
10151	* Dtrace tracepoints go first. We do them here at once so we don't
10152	* have to copy the guest state saving and stuff a few dozen times.
10153	* Down side is that we've got to repeat the switch, though this time
10154	* we use enmEvent since the probes are a subset of what DBGF does.
10155	*/
10156	if (fDtrace1 \|\| fDtrace2)
10157	{
10158	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10159	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10160	switch (enmEvent1)
10161	{
10162	/** @todo consider which extra parameters would be helpful for each probe. */
10163	case DBGFEVENT_END: break;
10164	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10165	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10166	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10167	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10168	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10169	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10170	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10171	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10172	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10173	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10174	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10175	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10176	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10177	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10178	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10179	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10180	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10181	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10182	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10183	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10184	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10185	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10186	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10187	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10188	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10189	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10190	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10191	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10192	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10193	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10194	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10195	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10196	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10197	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10198	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10199	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10200	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10201	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10202	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10203	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10204	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10205	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10206	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10207	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10208	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10209	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10210	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10211	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10212	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10213	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10214	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10215	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10216	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10217	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10218	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10219	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10220	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10221	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10222	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10223	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10224	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10225	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10226	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10227	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10228	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10229	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10230	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10231	}
10232	switch (enmEvent2)
10233	{
10234	/** @todo consider which extra parameters would be helpful for each probe. */
10235	case DBGFEVENT_END: break;
10236	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10237	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10238	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10239	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10240	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10241	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10242	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10243	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10244	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10245	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10246	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10247	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10248	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10249	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10250	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10251	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10252	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10253	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10254	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10255	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10256	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10257	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10258	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10259	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10260	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10261	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10262	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10263	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10264	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10265	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10266	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10267	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10268	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10269	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10270	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10271	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10272	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10273	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10274	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10275	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10276	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10277	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10278	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10279	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10280	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10281	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10282	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10283	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10284	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10285	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10286	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10287	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10288	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10289	}
10290	}
10291
10292	/*
10293	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10294	* the DBGF call will do a full check).
10295	*
10296	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10297	* Note! If we have to events, we prioritize the first, i.e. the instruction
10298	* one, in order to avoid event nesting.
10299	*/
10300	if ( enmEvent1 != DBGFEVENT_END
10301	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10302	{
10303	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10304	if (rcStrict != VINF_SUCCESS)
10305	return rcStrict;
10306	}
10307	else if ( enmEvent2 != DBGFEVENT_END
10308	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10309	{
10310	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10311	if (rcStrict != VINF_SUCCESS)
10312	return rcStrict;
10313	}
10314
10315	return VINF_SUCCESS;
10316	}
10317
10318
10319	/**
10320	* Single-stepping VM-exit filtering.
10321	*
10322	* This is preprocessing the VM-exits and deciding whether we've gotten far
10323	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10324	* handling is performed.
10325	*
10326	* @returns Strict VBox status code (i.e. informational status codes too).
10327	* @param pVM The cross context VM structure.
10328	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10329	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10330	* out-of-sync. Make sure to update the required
10331	* fields before using them.
10332	* @param pVmxTransient Pointer to the VMX-transient structure.
10333	* @param uExitReason The VM-exit reason.
10334	* @param pDbgState The debug state.
10335	*/
10336	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10337	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10338	{
10339	/*
10340	* Expensive (saves context) generic dtrace VM-exit probe.
10341	*/
10342	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10343	{ /* more likely */ }
10344	else
10345	{
10346	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10347	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10348	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10349	}
10350
10351	/*
10352	* Check for host NMI, just to get that out of the way.
10353	*/
10354	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10355	{ /* normally likely */ }
10356	else
10357	{
10358	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10359	AssertRCReturn(rc2, rc2);
10360	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10361	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10362	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10363	}
10364
10365	/*
10366	* Check for single stepping event if we're stepping.
10367	*/
10368	if (pVCpu->hm.s.fSingleInstruction)
10369	{
10370	switch (uExitReason)
10371	{
10372	case VMX_EXIT_MTF:
10373	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10374
10375	/* Various events: */
10376	case VMX_EXIT_XCPT_OR_NMI:
10377	case VMX_EXIT_EXT_INT:
10378	case VMX_EXIT_TRIPLE_FAULT:
10379	case VMX_EXIT_INT_WINDOW:
10380	case VMX_EXIT_NMI_WINDOW:
10381	case VMX_EXIT_TASK_SWITCH:
10382	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10383	case VMX_EXIT_APIC_ACCESS:
10384	case VMX_EXIT_EPT_VIOLATION:
10385	case VMX_EXIT_EPT_MISCONFIG:
10386	case VMX_EXIT_PREEMPT_TIMER:
10387
10388	/* Instruction specific VM-exits: */
10389	case VMX_EXIT_CPUID:
10390	case VMX_EXIT_GETSEC:
10391	case VMX_EXIT_HLT:
10392	case VMX_EXIT_INVD:
10393	case VMX_EXIT_INVLPG:
10394	case VMX_EXIT_RDPMC:
10395	case VMX_EXIT_RDTSC:
10396	case VMX_EXIT_RSM:
10397	case VMX_EXIT_VMCALL:
10398	case VMX_EXIT_VMCLEAR:
10399	case VMX_EXIT_VMLAUNCH:
10400	case VMX_EXIT_VMPTRLD:
10401	case VMX_EXIT_VMPTRST:
10402	case VMX_EXIT_VMREAD:
10403	case VMX_EXIT_VMRESUME:
10404	case VMX_EXIT_VMWRITE:
10405	case VMX_EXIT_VMXOFF:
10406	case VMX_EXIT_VMXON:
10407	case VMX_EXIT_MOV_CRX:
10408	case VMX_EXIT_MOV_DRX:
10409	case VMX_EXIT_IO_INSTR:
10410	case VMX_EXIT_RDMSR:
10411	case VMX_EXIT_WRMSR:
10412	case VMX_EXIT_MWAIT:
10413	case VMX_EXIT_MONITOR:
10414	case VMX_EXIT_PAUSE:
10415	case VMX_EXIT_XDTR_ACCESS:
10416	case VMX_EXIT_TR_ACCESS:
10417	case VMX_EXIT_INVEPT:
10418	case VMX_EXIT_RDTSCP:
10419	case VMX_EXIT_INVVPID:
10420	case VMX_EXIT_WBINVD:
10421	case VMX_EXIT_XSETBV:
10422	case VMX_EXIT_RDRAND:
10423	case VMX_EXIT_INVPCID:
10424	case VMX_EXIT_VMFUNC:
10425	case VMX_EXIT_RDSEED:
10426	case VMX_EXIT_XSAVES:
10427	case VMX_EXIT_XRSTORS:
10428	{
10429	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10430	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10431	AssertRCReturn(rc2, rc2);
10432	if ( pMixedCtx->rip != pDbgState->uRipStart
10433	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10434	return VINF_EM_DBG_STEPPED;
10435	break;
10436	}
10437
10438	/* Errors and unexpected events: */
10439	case VMX_EXIT_INIT_SIGNAL:
10440	case VMX_EXIT_SIPI:
10441	case VMX_EXIT_IO_SMI:
10442	case VMX_EXIT_SMI:
10443	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10444	case VMX_EXIT_ERR_MSR_LOAD:
10445	case VMX_EXIT_ERR_MACHINE_CHECK:
10446	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10447	break;
10448
10449	default:
10450	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10451	break;
10452	}
10453	}
10454
10455	/*
10456	* Check for debugger event breakpoints and dtrace probes.
10457	*/
10458	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10459	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10460	{
10461	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10462	if (rcStrict != VINF_SUCCESS)
10463	return rcStrict;
10464	}
10465
10466	/*
10467	* Normal processing.
10468	*/
10469	#ifdef HMVMX_USE_FUNCTION_TABLE
10470	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10471	#else
10472	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10473	#endif
10474	}
10475
10476
10477	/**
10478	* Single steps guest code using VT-x.
10479	*
10480	* @returns Strict VBox status code (i.e. informational status codes too).
10481	* @param pVM The cross context VM structure.
10482	* @param pVCpu The cross context virtual CPU structure.
10483	* @param pCtx Pointer to the guest-CPU context.
10484	*
10485	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10486	*/
10487	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10488	{
10489	VMXTRANSIENT VmxTransient;
10490	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10491
10492	/* Set HMCPU indicators. */
10493	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10494	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10495	pVCpu->hm.s.fDebugWantRdTscExit = false;
10496	pVCpu->hm.s.fUsingDebugLoop = true;
10497
10498	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10499	VMXRUNDBGSTATE DbgState;
10500	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10501	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10502
10503	/*
10504	* The loop.
10505	*/
10506	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10507	for (uint32_t cLoops = 0; ; cLoops++)
10508	{
10509	Assert(!HMR0SuspendPending());
10510	HMVMX_ASSERT_CPU_SAFE();
10511	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10512
10513	/*
10514	* Preparatory work for running guest code, this may force us to return
10515	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10516	*/
10517	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10518	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10519	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10520	if (rcStrict != VINF_SUCCESS)
10521	break;
10522
10523	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10524	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10525
10526	/*
10527	* Now we can run the guest code.
10528	*/
10529	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10530
10531	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10532
10533	/*
10534	* Restore any residual host-state and save any bits shared between host
10535	* and guest into the guest-CPU state. Re-enables interrupts!
10536	*/
10537	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10538
10539	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10540	if (RT_SUCCESS(rcRun))
10541	{ /* very likely */ }
10542	else
10543	{
10544	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10545	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10546	return rcRun;
10547	}
10548
10549	/* Profile the VM-exit. */
10550	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10551	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10552	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10553	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10554	HMVMX_START_EXIT_DISPATCH_PROF();
10555
10556	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10557
10558	/*
10559	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10560	*/
10561	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10562	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10563	if (rcStrict != VINF_SUCCESS)
10564	break;
10565	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10566	{
10567	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10568	rcStrict = VINF_EM_RAW_INTERRUPT;
10569	break;
10570	}
10571
10572	/*
10573	* Stepping: Did the RIP change, if so, consider it a single step.
10574	* Otherwise, make sure one of the TFs gets set.
10575	*/
10576	if (fStepping)
10577	{
10578	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10579	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10580	AssertRCReturn(rc2, rc2);
10581	if ( pCtx->rip != DbgState.uRipStart
10582	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10583	{
10584	rcStrict = VINF_EM_DBG_STEPPED;
10585	break;
10586	}
10587	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10588	}
10589
10590	/*
10591	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10592	*/
10593	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10594	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10595	}
10596
10597	/*
10598	* Clear the X86_EFL_TF if necessary.
10599	*/
10600	if (pVCpu->hm.s.fClearTrapFlag)
10601	{
10602	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10603	AssertRCReturn(rc2, rc2);
10604	pVCpu->hm.s.fClearTrapFlag = false;
10605	pCtx->eflags.Bits.u1TF = 0;
10606	}
10607	/** @todo there seems to be issues with the resume flag when the monitor trap
10608	* flag is pending without being used. Seen early in bios init when
10609	* accessing APIC page in protected mode. */
10610
10611	/*
10612	* Restore VM-exit control settings as we may not reenter this function the
10613	* next time around.
10614	*/
10615	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10616
10617	/* Restore HMCPU indicators. */
10618	pVCpu->hm.s.fUsingDebugLoop = false;
10619	pVCpu->hm.s.fDebugWantRdTscExit = false;
10620	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10621
10622	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10623	return rcStrict;
10624	}
10625
10626
10627	/** @} */
10628
10629
10630	/**
10631	* Checks if any expensive dtrace probes are enabled and we should go to the
10632	* debug loop.
10633	*
10634	* @returns true if we should use debug loop, false if not.
10635	*/
10636	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10637	{
10638	/* It's probably faster to OR the raw 32-bit counter variables together.
10639	Since the variables are in an array and the probes are next to one
10640	another (more or less), we have good locality. So, better read
10641	eight-nine cache lines ever time and only have one conditional, than
10642	128+ conditionals, right? */
10643	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10644	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10645	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10646	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10647	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10648	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10649	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10650	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10651	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10652	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10653	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10654	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10655	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10656	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10657	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10658	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10659	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10660	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10661	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10662	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10663	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10664	) != 0
10665	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10666	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10667	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10668	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10669	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10670	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10671	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10672	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10673	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10674	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10675	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10676	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10677	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10678	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10679	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10680	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10681	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10682	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10683	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10684	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10685	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10686	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10687	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10688	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10689	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10690	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10691	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10692	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10693	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10694	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10695	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10696	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10697	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10698	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10699	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10700	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10701	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10702	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10703	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10704	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10705	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10706	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10707	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10708	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10709	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10710	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10711	) != 0
10712	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10713	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10714	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10715	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10716	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10717	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10718	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10719	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10720	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10721	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10722	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10723	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10724	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10725	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10726	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10727	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10728	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10729	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10730	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10731	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10732	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10733	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10734	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10735	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10736	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10737	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10738	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10739	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10740	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10741	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10742	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10743	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10744	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10745	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10746	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10747	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10748	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10749	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10750	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10751	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10752	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10753	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10754	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10755	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10756	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10757	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10758	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10759	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10760	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10761	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10762	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10763	) != 0;
10764	}
10765
10766
10767	/**
10768	* Runs the guest code using VT-x.
10769	*
10770	* @returns Strict VBox status code (i.e. informational status codes too).
10771	* @param pVM The cross context VM structure.
10772	* @param pVCpu The cross context virtual CPU structure.
10773	* @param pCtx Pointer to the guest-CPU context.
10774	*/
10775	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10776	{
10777	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10778	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10779	HMVMX_ASSERT_PREEMPT_SAFE();
10780
10781	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10782
10783	VBOXSTRICTRC rcStrict;
10784	if ( !pVCpu->hm.s.fUseDebugLoop
10785	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10786	&& !DBGFIsStepping(pVCpu)
10787	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10788	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10789	else
10790	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10791
10792	if (rcStrict == VERR_EM_INTERPRETER)
10793	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10794	else if (rcStrict == VINF_EM_RESET)
10795	rcStrict = VINF_EM_TRIPLE_FAULT;
10796
10797	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10798	if (RT_FAILURE(rc2))
10799	{
10800	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10801	rcStrict = rc2;
10802	}
10803	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10804	return rcStrict;
10805	}
10806
10807
10808	#ifndef HMVMX_USE_FUNCTION_TABLE
10809	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10810	{
10811	# ifdef DEBUG_ramshankar
10812	# define RETURN_EXIT_CALL(a_CallExpr) \
10813	do { \
10814	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10815	VBOXSTRICTRC rcStrict = a_CallExpr; \
10816	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10817	return rcStrict; \
10818	} while (0)
10819	# else
10820	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10821	# endif
10822	switch (rcReason)
10823	{
10824	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10825	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10826	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10827	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10828	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10829	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10830	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10831	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10832	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10833	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10834	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10835	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10836	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10837	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10838	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10839	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10840	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10841	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10842	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10843	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10844	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10845	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10846	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10847	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10848	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10849	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10850	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10851	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10852	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10853	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10854	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10855	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10856	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10857	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10858
10859	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10860	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10861	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10862	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10863	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10864	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10865	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10866	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10867	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10868
10869	case VMX_EXIT_VMCLEAR:
10870	case VMX_EXIT_VMLAUNCH:
10871	case VMX_EXIT_VMPTRLD:
10872	case VMX_EXIT_VMPTRST:
10873	case VMX_EXIT_VMREAD:
10874	case VMX_EXIT_VMRESUME:
10875	case VMX_EXIT_VMWRITE:
10876	case VMX_EXIT_VMXOFF:
10877	case VMX_EXIT_VMXON:
10878	case VMX_EXIT_INVEPT:
10879	case VMX_EXIT_INVVPID:
10880	case VMX_EXIT_VMFUNC:
10881	case VMX_EXIT_XSAVES:
10882	case VMX_EXIT_XRSTORS:
10883	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10884	case VMX_EXIT_ENCLS:
10885	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10886	case VMX_EXIT_PML_FULL:
10887	default:
10888	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10889	}
10890	#undef RETURN_EXIT_CALL
10891	}
10892	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10893
10894
10895	#ifdef VBOX_STRICT
10896	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10897	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10898	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10899
10900	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10901	do { \
10902	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10903	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10904	} while (0)
10905
10906	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10907	do { \
10908	AssertPtr(pVCpu); \
10909	AssertPtr(pMixedCtx); \
10910	AssertPtr(pVmxTransient); \
10911	Assert(pVmxTransient->fVMEntryFailed == false); \
10912	Assert(ASMIntAreEnabled()); \
10913	HMVMX_ASSERT_PREEMPT_SAFE(); \
10914	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10915	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10916	HMVMX_ASSERT_PREEMPT_SAFE(); \
10917	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10918	HMVMX_ASSERT_PREEMPT_CPUID(); \
10919	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10920	} while (0)
10921
10922	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10923	do { \
10924	Log4Func(("\n")); \
10925	} while (0)
10926	#else /* nonstrict builds: */
10927	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10928	do { \
10929	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10930	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10931	} while (0)
10932	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10933	#endif
10934
10935
10936	/**
10937	* Advances the guest RIP by the specified number of bytes.
10938	*
10939	* @param pVCpu The cross context virtual CPU structure.
10940	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10941	* out-of-sync. Make sure to update the required fields
10942	* before using them.
10943	* @param cbInstr Number of bytes to advance the RIP by.
10944	*
10945	* @remarks No-long-jump zone!!!
10946	*/
10947	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10948	{
10949	/* Advance the RIP. */
10950	pMixedCtx->rip += cbInstr;
10951	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10952
10953	/* Update interrupt inhibition. */
10954	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10955	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10956	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10957	}
10958
10959
10960	/**
10961	* Advances the guest RIP after reading it from the VMCS.
10962	*
10963	* @returns VBox status code, no informational status codes.
10964	* @param pVCpu The cross context virtual CPU structure.
10965	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10966	* out-of-sync. Make sure to update the required fields
10967	* before using them.
10968	* @param pVmxTransient Pointer to the VMX transient structure.
10969	*
10970	* @remarks No-long-jump zone!!!
10971	*/
10972	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10973	{
10974	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10975	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10976	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10977	AssertRCReturn(rc, rc);
10978
10979	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10980
10981	/*
10982	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10983	* pending debug exception field as it takes care of priority of events.
10984	*
10985	* See Intel spec. 32.2.1 "Debug Exceptions".
10986	*/
10987	if ( !pVCpu->hm.s.fSingleInstruction
10988	&& pMixedCtx->eflags.Bits.u1TF)
10989	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10990
10991	return VINF_SUCCESS;
10992	}
10993
10994
10995	/**
10996	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10997	* and update error record fields accordingly.
10998	*
10999	* @return VMX_IGS_* return codes.
11000	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
11001	* wrong with the guest state.
11002	*
11003	* @param pVM The cross context VM structure.
11004	* @param pVCpu The cross context virtual CPU structure.
11005	* @param pCtx Pointer to the guest-CPU state.
11006	*
11007	* @remarks This function assumes our cache of the VMCS controls
11008	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
11009	*/
11010	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
11011	{
11012	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
11013	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
11014	uError = (err); \
11015	break; \
11016	} else do { } while (0)
11017
11018	int rc;
11019	uint32_t uError = VMX_IGS_ERROR;
11020	uint32_t u32Val;
11021	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
11022
11023	do
11024	{
11025	/*
11026	* CR0.
11027	*/
11028	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11029	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11030	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
11031	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
11032	if (fUnrestrictedGuest)
11033	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
11034
11035	uint32_t u32GuestCR0;
11036	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
11037	AssertRCBreak(rc);
11038	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
11039	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
11040	if ( !fUnrestrictedGuest
11041	&& (u32GuestCR0 & X86_CR0_PG)
11042	&& !(u32GuestCR0 & X86_CR0_PE))
11043	{
11044	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
11045	}
11046
11047	/*
11048	* CR4.
11049	*/
11050	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11051	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11052
11053	uint32_t u32GuestCR4;
11054	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11055	AssertRCBreak(rc);
11056	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11057	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11058
11059	/*
11060	* IA32_DEBUGCTL MSR.
11061	*/
11062	uint64_t u64Val;
11063	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11064	AssertRCBreak(rc);
11065	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11066	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11067	{
11068	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11069	}
11070	uint64_t u64DebugCtlMsr = u64Val;
11071
11072	#ifdef VBOX_STRICT
11073	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11074	AssertRCBreak(rc);
11075	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11076	#endif
11077	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11078
11079	/*
11080	* RIP and RFLAGS.
11081	*/
11082	uint32_t u32Eflags;
11083	#if HC_ARCH_BITS == 64
11084	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11085	AssertRCBreak(rc);
11086	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11087	if ( !fLongModeGuest
11088	\|\| !pCtx->cs.Attr.n.u1Long)
11089	{
11090	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11091	}
11092	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11093	* must be identical if the "IA-32e mode guest" VM-entry
11094	* control is 1 and CS.L is 1. No check applies if the
11095	* CPU supports 64 linear-address bits. */
11096
11097	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11098	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11099	AssertRCBreak(rc);
11100	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11101	VMX_IGS_RFLAGS_RESERVED);
11102	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11103	u32Eflags = u64Val;
11104	#else
11105	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11106	AssertRCBreak(rc);
11107	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11108	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11109	#endif
11110
11111	if ( fLongModeGuest
11112	\|\| ( fUnrestrictedGuest
11113	&& !(u32GuestCR0 & X86_CR0_PE)))
11114	{
11115	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11116	}
11117
11118	uint32_t u32EntryInfo;
11119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11120	AssertRCBreak(rc);
11121	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11122	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11123	{
11124	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11125	}
11126
11127	/*
11128	* 64-bit checks.
11129	*/
11130	#if HC_ARCH_BITS == 64
11131	if (fLongModeGuest)
11132	{
11133	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11134	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11135	}
11136
11137	if ( !fLongModeGuest
11138	&& (u32GuestCR4 & X86_CR4_PCIDE))
11139	{
11140	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11141	}
11142
11143	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11144	* 51:32 beyond the processor's physical-address width are 0. */
11145
11146	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11147	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11148	{
11149	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11150	}
11151
11152	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11153	AssertRCBreak(rc);
11154	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11155
11156	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11157	AssertRCBreak(rc);
11158	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11159	#endif
11160
11161	/*
11162	* PERF_GLOBAL MSR.
11163	*/
11164	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11165	{
11166	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11167	AssertRCBreak(rc);
11168	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11169	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11170	}
11171
11172	/*
11173	* PAT MSR.
11174	*/
11175	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11176	{
11177	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11178	AssertRCBreak(rc);
11179	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11180	for (unsigned i = 0; i < 8; i++)
11181	{
11182	uint8_t u8Val = (u64Val & 0xff);
11183	if ( u8Val != 0 /* UC */
11184	&& u8Val != 1 /* WC */
11185	&& u8Val != 4 /* WT */
11186	&& u8Val != 5 /* WP */
11187	&& u8Val != 6 /* WB */
11188	&& u8Val != 7 /* UC- */)
11189	{
11190	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11191	}
11192	u64Val >>= 8;
11193	}
11194	}
11195
11196	/*
11197	* EFER MSR.
11198	*/
11199	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11200	{
11201	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11202	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11203	AssertRCBreak(rc);
11204	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11205	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11206	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11207	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11208	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11209	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11210	\|\| !(u32GuestCR0 & X86_CR0_PG)
11211	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11212	VMX_IGS_EFER_LMA_LME_MISMATCH);
11213	}
11214
11215	/*
11216	* Segment registers.
11217	*/
11218	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11219	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11220	if (!(u32Eflags & X86_EFL_VM))
11221	{
11222	/* CS */
11223	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11224	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11225	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11226	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11227	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11228	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11229	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11230	/* CS cannot be loaded with NULL in protected mode. */
11231	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11232	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11233	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11234	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11235	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11236	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11237	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11238	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11239	else
11240	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11241
11242	/* SS */
11243	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11244	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11245	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11246	if ( !(pCtx->cr0 & X86_CR0_PE)
11247	\|\| pCtx->cs.Attr.n.u4Type == 3)
11248	{
11249	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11250	}
11251	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11252	{
11253	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11254	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11255	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11256	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11257	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11258	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11259	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11260	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11261	}
11262
11263	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11264	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11265	{
11266	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11267	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11268	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11269	\|\| pCtx->ds.Attr.n.u4Type > 11
11270	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11271	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11272	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11273	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11274	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11275	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11276	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11277	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11278	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11279	}
11280	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11281	{
11282	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11283	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11284	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11285	\|\| pCtx->es.Attr.n.u4Type > 11
11286	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11287	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11288	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11289	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11290	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11291	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11292	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11293	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11294	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11295	}
11296	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11297	{
11298	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11299	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11300	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11301	\|\| pCtx->fs.Attr.n.u4Type > 11
11302	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11303	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11304	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11305	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11306	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11307	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11308	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11309	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11310	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11311	}
11312	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11313	{
11314	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11315	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11316	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11317	\|\| pCtx->gs.Attr.n.u4Type > 11
11318	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11319	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11320	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11321	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11322	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11323	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11324	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11325	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11326	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11327	}
11328	/* 64-bit capable CPUs. */
11329	#if HC_ARCH_BITS == 64
11330	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11331	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11332	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11333	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11334	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11335	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11336	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11337	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11338	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11339	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11340	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11341	#endif
11342	}
11343	else
11344	{
11345	/* V86 mode checks. */
11346	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11347	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11348	{
11349	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11350	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11351	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11352	}
11353	else
11354	{
11355	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11356	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11357	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11358	}
11359
11360	/* CS */
11361	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11362	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11363	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11364	/* SS */
11365	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11366	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11367	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11368	/* DS */
11369	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11370	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11371	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11372	/* ES */
11373	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11374	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11375	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11376	/* FS */
11377	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11378	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11379	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11380	/* GS */
11381	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11382	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11383	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11384	/* 64-bit capable CPUs. */
11385	#if HC_ARCH_BITS == 64
11386	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11387	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11388	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11389	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11390	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11391	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11392	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11393	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11394	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11395	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11396	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11397	#endif
11398	}
11399
11400	/*
11401	* TR.
11402	*/
11403	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11404	/* 64-bit capable CPUs. */
11405	#if HC_ARCH_BITS == 64
11406	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11407	#endif
11408	if (fLongModeGuest)
11409	{
11410	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11411	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11412	}
11413	else
11414	{
11415	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11416	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11417	VMX_IGS_TR_ATTR_TYPE_INVALID);
11418	}
11419	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11420	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11421	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11422	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11423	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11424	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11425	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11426	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11427
11428	/*
11429	* GDTR and IDTR.
11430	*/
11431	#if HC_ARCH_BITS == 64
11432	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11433	AssertRCBreak(rc);
11434	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11435
11436	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11437	AssertRCBreak(rc);
11438	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11439	#endif
11440
11441	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11442	AssertRCBreak(rc);
11443	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11444
11445	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11446	AssertRCBreak(rc);
11447	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11448
11449	/*
11450	* Guest Non-Register State.
11451	*/
11452	/* Activity State. */
11453	uint32_t u32ActivityState;
11454	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11455	AssertRCBreak(rc);
11456	HMVMX_CHECK_BREAK( !u32ActivityState
11457	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11458	VMX_IGS_ACTIVITY_STATE_INVALID);
11459	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11460	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11461	uint32_t u32IntrState;
11462	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11463	AssertRCBreak(rc);
11464	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11465	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11466	{
11467	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11468	}
11469
11470	/** @todo Activity state and injecting interrupts. Left as a todo since we
11471	* currently don't use activity states but ACTIVE. */
11472
11473	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11474	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11475
11476	/* Guest interruptibility-state. */
11477	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11478	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11479	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11480	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11481	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11482	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11483	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11484	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11485	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11486	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11487	{
11488	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11489	{
11490	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11491	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11492	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11493	}
11494	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11495	{
11496	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11497	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11498	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11499	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11500	}
11501	}
11502	/** @todo Assumes the processor is not in SMM. */
11503	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11504	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11505	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11506	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11507	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11508	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11509	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11510	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11511	{
11512	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11513	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11514	}
11515
11516	/* Pending debug exceptions. */
11517	#if HC_ARCH_BITS == 64
11518	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11519	AssertRCBreak(rc);
11520	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11521	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11522	u32Val = u64Val; /* For pending debug exceptions checks below. */
11523	#else
11524	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11525	AssertRCBreak(rc);
11526	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11527	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11528	#endif
11529
11530	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11531	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11532	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11533	{
11534	if ( (u32Eflags & X86_EFL_TF)
11535	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11536	{
11537	/* Bit 14 is PendingDebug.BS. */
11538	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11539	}
11540	if ( !(u32Eflags & X86_EFL_TF)
11541	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11542	{
11543	/* Bit 14 is PendingDebug.BS. */
11544	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11545	}
11546	}
11547
11548	/* VMCS link pointer. */
11549	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11550	AssertRCBreak(rc);
11551	if (u64Val != UINT64_C(0xffffffffffffffff))
11552	{
11553	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11554	/** @todo Bits beyond the processor's physical-address width MBZ. */
11555	/** @todo 32-bit located in memory referenced by value of this field (as a
11556	* physical address) must contain the processor's VMCS revision ID. */
11557	/** @todo SMM checks. */
11558	}
11559
11560	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11561	* not using Nested Paging? */
11562	if ( pVM->hm.s.fNestedPaging
11563	&& !fLongModeGuest
11564	&& CPUMIsGuestInPAEModeEx(pCtx))
11565	{
11566	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11567	AssertRCBreak(rc);
11568	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11569
11570	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11571	AssertRCBreak(rc);
11572	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11573
11574	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11575	AssertRCBreak(rc);
11576	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11577
11578	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11579	AssertRCBreak(rc);
11580	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11581	}
11582
11583	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11584	if (uError == VMX_IGS_ERROR)
11585	uError = VMX_IGS_REASON_NOT_FOUND;
11586	} while (0);
11587
11588	pVCpu->hm.s.u32HMError = uError;
11589	return uError;
11590
11591	#undef HMVMX_ERROR_BREAK
11592	#undef HMVMX_CHECK_BREAK
11593	}
11594
11595	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11596	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11597	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11598
11599	/** @name VM-exit handlers.
11600	* @{
11601	*/
11602
11603	/**
11604	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11605	*/
11606	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11607	{
11608	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11610	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11611	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11612	return VINF_SUCCESS;
11613	return VINF_EM_RAW_INTERRUPT;
11614	}
11615
11616
11617	/**
11618	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11619	*/
11620	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11621	{
11622	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11623	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11624
11625	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11626	AssertRCReturn(rc, rc);
11627
11628	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11629	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11630	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11631	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11632
11633	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11634	{
11635	/*
11636	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11637	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11638	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11639	*
11640	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11641	*/
11642	VMXDispatchHostNmi();
11643	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11644	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11645	return VINF_SUCCESS;
11646	}
11647
11648	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11649	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11650	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11651	{ /* likely */ }
11652	else
11653	{
11654	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11655	rcStrictRc1 = VINF_SUCCESS;
11656	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11657	return rcStrictRc1;
11658	}
11659
11660	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11661	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11662	switch (uIntType)
11663	{
11664	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11665	Assert(uVector == X86_XCPT_DB);
11666	RT_FALL_THRU();
11667	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11668	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11669	RT_FALL_THRU();
11670	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11671	{
11672	/*
11673	* If there's any exception caused as a result of event injection, the resulting
11674	* secondary/final execption will be pending, we shall continue guest execution
11675	* after injecting the event. The page-fault case is complicated and we manually
11676	* handle any currently pending event in hmR0VmxExitXcptPF.
11677	*/
11678	if (!pVCpu->hm.s.Event.fPending)
11679	{ /* likely */ }
11680	else if (uVector != X86_XCPT_PF)
11681	{
11682	rc = VINF_SUCCESS;
11683	break;
11684	}
11685
11686	switch (uVector)
11687	{
11688	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11689	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11690	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11691	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11692	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11693	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11694	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11695
11696	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11697	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11698	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11699	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11700	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11701	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11702	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11703	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11704	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11705	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11706	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11707	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11708	default:
11709	{
11710	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11711	AssertRCReturn(rc, rc);
11712
11713	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11714	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11715	{
11716	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11717	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11718	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11719
11720	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11721	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11722	AssertRCReturn(rc, rc);
11723	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11724	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11725	0 /* GCPtrFaultAddress */);
11726	AssertRCReturn(rc, rc);
11727	}
11728	else
11729	{
11730	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11731	pVCpu->hm.s.u32HMError = uVector;
11732	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11733	}
11734	break;
11735	}
11736	}
11737	break;
11738	}
11739
11740	default:
11741	{
11742	pVCpu->hm.s.u32HMError = uExitIntInfo;
11743	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11744	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11745	break;
11746	}
11747	}
11748	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11749	return rc;
11750	}
11751
11752
11753	/**
11754	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11755	*/
11756	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11757	{
11758	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11759
11760	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11761	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11762
11763	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11764	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11765	return VINF_SUCCESS;
11766	}
11767
11768
11769	/**
11770	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11771	*/
11772	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11773	{
11774	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11775	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11776	{
11777	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11778	HMVMX_RETURN_UNEXPECTED_EXIT();
11779	}
11780
11781	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11782
11783	/*
11784	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11785	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11786	*/
11787	uint32_t uIntrState = 0;
11788	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11789	AssertRCReturn(rc, rc);
11790
11791	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11792	if ( fBlockSti
11793	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11794	{
11795	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11796	}
11797
11798	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11799	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11800
11801	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11802	return VINF_SUCCESS;
11803	}
11804
11805
11806	/**
11807	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11808	*/
11809	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11810	{
11811	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11812	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11813	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11814	}
11815
11816
11817	/**
11818	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11819	*/
11820	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11821	{
11822	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11823	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11824	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11825	}
11826
11827
11828	/**
11829	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11830	*/
11831	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11832	{
11833	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11834	PVM pVM = pVCpu->CTX_SUFF(pVM);
11835	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11836	if (RT_LIKELY(rc == VINF_SUCCESS))
11837	{
11838	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11839	Assert(pVmxTransient->cbInstr == 2);
11840	}
11841	else
11842	{
11843	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11844	rc = VERR_EM_INTERPRETER;
11845	}
11846	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11847	return rc;
11848	}
11849
11850
11851	/**
11852	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11853	*/
11854	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11855	{
11856	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11857	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11858	AssertRCReturn(rc, rc);
11859
11860	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11861	return VINF_EM_RAW_EMULATE_INSTR;
11862
11863	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11864	HMVMX_RETURN_UNEXPECTED_EXIT();
11865	}
11866
11867
11868	/**
11869	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11870	*/
11871	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11872	{
11873	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11874	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11875	AssertRCReturn(rc, rc);
11876
11877	PVM pVM = pVCpu->CTX_SUFF(pVM);
11878	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11879	if (RT_LIKELY(rc == VINF_SUCCESS))
11880	{
11881	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11882	Assert(pVmxTransient->cbInstr == 2);
11883	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11884	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11885	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11886	}
11887	else
11888	rc = VERR_EM_INTERPRETER;
11889	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11890	return rc;
11891	}
11892
11893
11894	/**
11895	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11896	*/
11897	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11898	{
11899	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11900	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11901	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11902	AssertRCReturn(rc, rc);
11903
11904	PVM pVM = pVCpu->CTX_SUFF(pVM);
11905	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11906	if (RT_SUCCESS(rc))
11907	{
11908	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11909	Assert(pVmxTransient->cbInstr == 3);
11910	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11911	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11912	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11913	}
11914	else
11915	{
11916	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11917	rc = VERR_EM_INTERPRETER;
11918	}
11919	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11920	return rc;
11921	}
11922
11923
11924	/**
11925	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11926	*/
11927	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11928	{
11929	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11930	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11931	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11932	AssertRCReturn(rc, rc);
11933
11934	PVM pVM = pVCpu->CTX_SUFF(pVM);
11935	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11936	if (RT_LIKELY(rc == VINF_SUCCESS))
11937	{
11938	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11939	Assert(pVmxTransient->cbInstr == 2);
11940	}
11941	else
11942	{
11943	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11944	rc = VERR_EM_INTERPRETER;
11945	}
11946	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11947	return rc;
11948	}
11949
11950
11951	/**
11952	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11953	*/
11954	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11955	{
11956	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11958
11959	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11960	if (pVCpu->hm.s.fHypercallsEnabled)
11961	{
11962	#if 0
11963	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11964	#else
11965	/* Aggressive state sync. for now. */
11966	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11967	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11968	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11969	AssertRCReturn(rc, rc);
11970	#endif
11971
11972	/* Perform the hypercall. */
11973	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11974	if (rcStrict == VINF_SUCCESS)
11975	{
11976	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11977	AssertRCReturn(rc, rc);
11978	}
11979	else
11980	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11981	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11982	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11983
11984	/* If the hypercall changes anything other than guest's general-purpose registers,
11985	we would need to reload the guest changed bits here before VM-entry. */
11986	}
11987	else
11988	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11989
11990	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11991	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11992	{
11993	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11994	rcStrict = VINF_SUCCESS;
11995	}
11996
11997	return rcStrict;
11998	}
11999
12000
12001	/**
12002	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
12003	*/
12004	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12005	{
12006	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12007	PVM pVM = pVCpu->CTX_SUFF(pVM);
12008	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
12009
12010	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12011	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12012	AssertRCReturn(rc, rc);
12013
12014	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
12015	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12016	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12017	else
12018	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
12019	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
12020	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
12021	return rcStrict;
12022	}
12023
12024
12025	/**
12026	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
12027	*/
12028	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12029	{
12030	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12031	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12032	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12033	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12034	AssertRCReturn(rc, rc);
12035
12036	PVM pVM = pVCpu->CTX_SUFF(pVM);
12037	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12038	if (RT_LIKELY(rc == VINF_SUCCESS))
12039	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12040	else
12041	{
12042	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
12043	rc = VERR_EM_INTERPRETER;
12044	}
12045	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
12046	return rc;
12047	}
12048
12049
12050	/**
12051	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12052	*/
12053	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12054	{
12055	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12056	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12057	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12058	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12059	AssertRCReturn(rc, rc);
12060
12061	PVM pVM = pVCpu->CTX_SUFF(pVM);
12062	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12063	rc = VBOXSTRICTRC_VAL(rc2);
12064	if (RT_LIKELY( rc == VINF_SUCCESS
12065	\|\| rc == VINF_EM_HALT))
12066	{
12067	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12068	AssertRCReturn(rc3, rc3);
12069
12070	if ( rc == VINF_EM_HALT
12071	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12072	{
12073	rc = VINF_SUCCESS;
12074	}
12075	}
12076	else
12077	{
12078	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12079	rc = VERR_EM_INTERPRETER;
12080	}
12081	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
12082	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12084	return rc;
12085	}
12086
12087
12088	/**
12089	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12090	*/
12091	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12092	{
12093	/*
12094	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12095	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12096	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
12097	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12098	*/
12099	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12100	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12101	HMVMX_RETURN_UNEXPECTED_EXIT();
12102	}
12103
12104
12105	/**
12106	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12107	*/
12108	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12109	{
12110	/*
12111	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12112	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12113	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12114	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12115	*/
12116	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12117	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12118	HMVMX_RETURN_UNEXPECTED_EXIT();
12119	}
12120
12121
12122	/**
12123	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12124	*/
12125	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12126	{
12127	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12128	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12129	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12130	HMVMX_RETURN_UNEXPECTED_EXIT();
12131	}
12132
12133
12134	/**
12135	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12136	*/
12137	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12138	{
12139	/*
12140	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12141	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12142	* See Intel spec. 25.3 "Other Causes of VM-exits".
12143	*/
12144	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12145	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12146	HMVMX_RETURN_UNEXPECTED_EXIT();
12147	}
12148
12149
12150	/**
12151	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12152	* VM-exit.
12153	*/
12154	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12155	{
12156	/*
12157	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12158	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12159	*
12160	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12161	* See Intel spec. "23.8 Restrictions on VMX operation".
12162	*/
12163	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12164	return VINF_SUCCESS;
12165	}
12166
12167
12168	/**
12169	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12170	* VM-exit.
12171	*/
12172	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12173	{
12174	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12175	return VINF_EM_RESET;
12176	}
12177
12178
12179	/**
12180	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12181	*/
12182	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12183	{
12184	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12185	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12186
12187	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12188	AssertRCReturn(rc, rc);
12189
12190	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12191	rc = VINF_SUCCESS;
12192	else
12193	rc = VINF_EM_HALT;
12194
12195	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12196	if (rc != VINF_SUCCESS)
12197	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12198	return rc;
12199	}
12200
12201
12202	/**
12203	* VM-exit handler for instructions that result in a \#UD exception delivered to
12204	* the guest.
12205	*/
12206	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12207	{
12208	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12209	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12210	return VINF_SUCCESS;
12211	}
12212
12213
12214	/**
12215	* VM-exit handler for expiry of the VMX preemption timer.
12216	*/
12217	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12218	{
12219	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12220
12221	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12222	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12223
12224	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12225	PVM pVM = pVCpu->CTX_SUFF(pVM);
12226	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12228	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12229	}
12230
12231
12232	/**
12233	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12234	*/
12235	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12236	{
12237	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12238
12239	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12240	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12241	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12242	AssertRCReturn(rc, rc);
12243
12244	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12245	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12246
12247	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12248
12249	return rcStrict;
12250	}
12251
12252
12253	/**
12254	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12255	*/
12256	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12257	{
12258	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12259	/** @todo Use VM-exit instruction information. */
12260	return VERR_EM_INTERPRETER;
12261	}
12262
12263
12264	/**
12265	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12266	* Error VM-exit.
12267	*/
12268	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12269	{
12270	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12271	AssertRCReturn(rc, rc);
12272
12273	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12274	AssertRCReturn(rc, rc);
12275
12276	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12277	NOREF(uInvalidReason);
12278
12279	#ifdef VBOX_STRICT
12280	uint32_t uIntrState;
12281	RTHCUINTREG uHCReg;
12282	uint64_t u64Val;
12283	uint32_t u32Val;
12284
12285	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12286	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12287	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12288	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12289	AssertRCReturn(rc, rc);
12290
12291	Log4(("uInvalidReason %u\n", uInvalidReason));
12292	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12293	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12294	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12295	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12296
12297	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12298	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12299	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12300	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12301	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12302	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12303	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12304	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12305	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12306	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12307	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12308	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12309	#else
12310	NOREF(pVmxTransient);
12311	#endif
12312
12313	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12314	return VERR_VMX_INVALID_GUEST_STATE;
12315	}
12316
12317
12318	/**
12319	* VM-exit handler for VM-entry failure due to an MSR-load
12320	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12321	*/
12322	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12323	{
12324	NOREF(pVmxTransient);
12325	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12326	HMVMX_RETURN_UNEXPECTED_EXIT();
12327	}
12328
12329
12330	/**
12331	* VM-exit handler for VM-entry failure due to a machine-check event
12332	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12333	*/
12334	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12335	{
12336	NOREF(pVmxTransient);
12337	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12338	HMVMX_RETURN_UNEXPECTED_EXIT();
12339	}
12340
12341
12342	/**
12343	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12344	* theory.
12345	*/
12346	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12347	{
12348	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12349	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12350	return VERR_VMX_UNDEFINED_EXIT_CODE;
12351	}
12352
12353
12354	/**
12355	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12356	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12357	* Conditional VM-exit.
12358	*/
12359	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12360	{
12361	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12362
12363	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12364	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12365	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12366	return VERR_EM_INTERPRETER;
12367	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12368	HMVMX_RETURN_UNEXPECTED_EXIT();
12369	}
12370
12371
12372	/**
12373	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12374	*/
12375	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12376	{
12377	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12378
12379	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12380	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12381	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12382	return VERR_EM_INTERPRETER;
12383	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12384	HMVMX_RETURN_UNEXPECTED_EXIT();
12385	}
12386
12387
12388	/**
12389	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12390	*/
12391	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12392	{
12393	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12394
12395	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12396	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12397	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12398	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12399	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12400	{
12401	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12402	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12403	}
12404	AssertRCReturn(rc, rc);
12405	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12406
12407	#ifdef VBOX_STRICT
12408	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12409	{
12410	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12411	&& pMixedCtx->ecx != MSR_K6_EFER)
12412	{
12413	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12414	pMixedCtx->ecx));
12415	HMVMX_RETURN_UNEXPECTED_EXIT();
12416	}
12417	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12418	{
12419	VMXMSREXITREAD enmRead;
12420	VMXMSREXITWRITE enmWrite;
12421	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12422	AssertRCReturn(rc2, rc2);
12423	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12424	{
12425	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12426	HMVMX_RETURN_UNEXPECTED_EXIT();
12427	}
12428	}
12429	}
12430	#endif
12431
12432	PVM pVM = pVCpu->CTX_SUFF(pVM);
12433	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12434	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12435	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12436	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12437	if (RT_SUCCESS(rc))
12438	{
12439	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12440	Assert(pVmxTransient->cbInstr == 2);
12441	}
12442	return rc;
12443	}
12444
12445
12446	/**
12447	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12448	*/
12449	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12450	{
12451	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12452	PVM pVM = pVCpu->CTX_SUFF(pVM);
12453	int rc = VINF_SUCCESS;
12454
12455	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12456	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12457	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12458	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12459	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12460	{
12461	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12462	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12463	}
12464	AssertRCReturn(rc, rc);
12465	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12466
12467	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12468	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12469	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12470
12471	if (RT_SUCCESS(rc))
12472	{
12473	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12474
12475	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12476	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12477	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12478	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12479	{
12480	/*
12481	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12482	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12483	* EMInterpretWrmsr() changes it.
12484	*/
12485	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12486	}
12487	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12488	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12489	else if (pMixedCtx->ecx == MSR_K6_EFER)
12490	{
12491	/*
12492	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12493	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12494	* the other bits as well, SCE and NXE. See @bugref{7368}.
12495	*/
12496	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12497	}
12498
12499	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12500	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12501	{
12502	switch (pMixedCtx->ecx)
12503	{
12504	/*
12505	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12506	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12507	*/
12508	case MSR_IA32_SYSENTER_CS:
12509	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12510	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12511	break;
12512	case MSR_IA32_SYSENTER_EIP:
12513	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12514	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12515	break;
12516	case MSR_IA32_SYSENTER_ESP:
12517	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12518	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12519	break;
12520	case MSR_K8_FS_BASE: RT_FALL_THRU();
12521	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12522	case MSR_K6_EFER: /* already handled above */ break;
12523	default:
12524	{
12525	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12526	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12527	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12528	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
12529	break;
12530	}
12531	}
12532	}
12533	#ifdef VBOX_STRICT
12534	else
12535	{
12536	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12537	switch (pMixedCtx->ecx)
12538	{
12539	case MSR_IA32_SYSENTER_CS:
12540	case MSR_IA32_SYSENTER_EIP:
12541	case MSR_IA32_SYSENTER_ESP:
12542	case MSR_K8_FS_BASE:
12543	case MSR_K8_GS_BASE:
12544	{
12545	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12546	HMVMX_RETURN_UNEXPECTED_EXIT();
12547	}
12548
12549	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12550	default:
12551	{
12552	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12553	{
12554	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12555	if (pMixedCtx->ecx != MSR_K6_EFER)
12556	{
12557	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12558	pMixedCtx->ecx));
12559	HMVMX_RETURN_UNEXPECTED_EXIT();
12560	}
12561	}
12562
12563	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12564	{
12565	VMXMSREXITREAD enmRead;
12566	VMXMSREXITWRITE enmWrite;
12567	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12568	AssertRCReturn(rc2, rc2);
12569	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12570	{
12571	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12572	HMVMX_RETURN_UNEXPECTED_EXIT();
12573	}
12574	}
12575	break;
12576	}
12577	}
12578	}
12579	#endif /* VBOX_STRICT */
12580	}
12581	return rc;
12582	}
12583
12584
12585	/**
12586	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12587	*/
12588	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12589	{
12590	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12591
12592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12593	return VINF_EM_RAW_INTERRUPT;
12594	}
12595
12596
12597	/**
12598	* VM-exit handler for when the TPR value is lowered below the specified
12599	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12600	*/
12601	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12602	{
12603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12604	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12605
12606	/*
12607	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12608	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12609	*/
12610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12611	return VINF_SUCCESS;
12612	}
12613
12614
12615	/**
12616	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12617	* VM-exit.
12618	*
12619	* @retval VINF_SUCCESS when guest execution can continue.
12620	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12621	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12622	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12623	* interpreter.
12624	*/
12625	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12626	{
12627	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12628	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12629	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12630	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12631	AssertRCReturn(rc, rc);
12632
12633	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12634	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12635	PVM pVM = pVCpu->CTX_SUFF(pVM);
12636	VBOXSTRICTRC rcStrict;
12637	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12638	switch (uAccessType)
12639	{
12640	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12641	{
12642	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12643	AssertRCReturn(rc, rc);
12644
12645	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12646	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12647	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12648	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12649	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12650	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12651	{
12652	case 0: /* CR0 */
12653	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12654	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12655	break;
12656	case 2: /* CR2 */
12657	/* Nothing to do here, CR2 it's not part of the VMCS. */
12658	break;
12659	case 3: /* CR3 */
12660	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12661	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12662	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12663	break;
12664	case 4: /* CR4 */
12665	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12666	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12667	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12668	break;
12669	case 8: /* CR8 */
12670	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12671	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12672	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12673	break;
12674	default:
12675	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12676	break;
12677	}
12678
12679	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12680	break;
12681	}
12682
12683	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12684	{
12685	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12686	AssertRCReturn(rc, rc);
12687
12688	Assert( !pVM->hm.s.fNestedPaging
12689	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12690	\|\| pVCpu->hm.s.fUsingDebugLoop
12691	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12692
12693	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12694	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12695	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12696
12697	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12698	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12699	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12700	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12701	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12702	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12703	VBOXSTRICTRC_VAL(rcStrict)));
12704	if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12705	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
12706	break;
12707	}
12708
12709	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12710	{
12711	AssertRCReturn(rc, rc);
12712	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12713	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12714	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12715	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12716	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12717	break;
12718	}
12719
12720	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12721	{
12722	AssertRCReturn(rc, rc);
12723	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12724	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12725	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12726	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12727	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12728	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12729	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12730	break;
12731	}
12732
12733	default:
12734	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12735	VERR_VMX_UNEXPECTED_EXCEPTION);
12736	}
12737
12738	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12739	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12740	NOREF(pVM);
12741	return rcStrict;
12742	}
12743
12744
12745	/**
12746	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12747	* VM-exit.
12748	*/
12749	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12750	{
12751	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12752	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12753
12754	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12755	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12756	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12757	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12758	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12759	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12760	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12761	AssertRCReturn(rc, rc);
12762
12763	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12764	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12765	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12766	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12767	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12768	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12769	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12770	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12771	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12772
12773	/* I/O operation lookup arrays. */
12774	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12775	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12776
12777	VBOXSTRICTRC rcStrict;
12778	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12779	uint32_t const cbInstr = pVmxTransient->cbInstr;
12780	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12781	PVM pVM = pVCpu->CTX_SUFF(pVM);
12782	if (fIOString)
12783	{
12784	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12785	See @bugref{5752#c158}. Should work now. */
12786	/*
12787	* INS/OUTS - I/O String instruction.
12788	*
12789	* Use instruction-information if available, otherwise fall back on
12790	* interpreting the instruction.
12791	*/
12792	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12793	fIOWrite ? 'w' : 'r'));
12794	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12795	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12796	{
12797	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12798	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12799	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12800	AssertRCReturn(rc2, rc2);
12801	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12802	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12803	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12804	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12805	if (fIOWrite)
12806	{
12807	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12808	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12809	}
12810	else
12811	{
12812	/*
12813	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12814	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12815	* See Intel Instruction spec. for "INS".
12816	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12817	*/
12818	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12819	}
12820	}
12821	else
12822	{
12823	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12824	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12825	AssertRCReturn(rc2, rc2);
12826	rcStrict = IEMExecOne(pVCpu);
12827	}
12828	/** @todo IEM needs to be setting these flags somehow. */
12829	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12830	fUpdateRipAlready = true;
12831	#else
12832	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12833	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12834	if (RT_SUCCESS(rcStrict))
12835	{
12836	if (fIOWrite)
12837	{
12838	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12839	(DISCPUMODE)pDis->uAddrMode, cbValue);
12840	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12841	}
12842	else
12843	{
12844	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12845	(DISCPUMODE)pDis->uAddrMode, cbValue);
12846	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12847	}
12848	}
12849	else
12850	{
12851	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12852	pMixedCtx->rip));
12853	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12854	}
12855	#endif
12856	}
12857	else
12858	{
12859	/*
12860	* IN/OUT - I/O instruction.
12861	*/
12862	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12863	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12864	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12865	if (fIOWrite)
12866	{
12867	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12869	}
12870	else
12871	{
12872	uint32_t u32Result = 0;
12873	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12874	if (IOM_SUCCESS(rcStrict))
12875	{
12876	/* Save result of I/O IN instr. in AL/AX/EAX. */
12877	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12878	}
12879	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12880	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12881	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12882	}
12883	}
12884
12885	if (IOM_SUCCESS(rcStrict))
12886	{
12887	if (!fUpdateRipAlready)
12888	{
12889	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12890	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12891	}
12892
12893	/*
12894	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12895	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12896	*/
12897	if (fIOString)
12898	{
12899	/** @todo Single-step for INS/OUTS with REP prefix? */
12900	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12901	}
12902	else if ( !fDbgStepping
12903	&& fGstStepping)
12904	{
12905	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12906	}
12907
12908	/*
12909	* If any I/O breakpoints are armed, we need to check if one triggered
12910	* and take appropriate action.
12911	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12912	*/
12913	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12914	AssertRCReturn(rc2, rc2);
12915
12916	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12917	* execution engines about whether hyper BPs and such are pending. */
12918	uint32_t const uDr7 = pMixedCtx->dr[7];
12919	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12920	&& X86_DR7_ANY_RW_IO(uDr7)
12921	&& (pMixedCtx->cr4 & X86_CR4_DE))
12922	\|\| DBGFBpIsHwIoArmed(pVM)))
12923	{
12924	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12925
12926	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12927	VMMRZCallRing3Disable(pVCpu);
12928	HM_DISABLE_PREEMPT();
12929
12930	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12931
12932	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12933	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12934	{
12935	/* Raise #DB. */
12936	if (fIsGuestDbgActive)
12937	ASMSetDR6(pMixedCtx->dr[6]);
12938	if (pMixedCtx->dr[7] != uDr7)
12939	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12940
12941	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12942	}
12943	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12944	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12945	else if ( rcStrict2 != VINF_SUCCESS
12946	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12947	rcStrict = rcStrict2;
12948	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12949
12950	HM_RESTORE_PREEMPT();
12951	VMMRZCallRing3Enable(pVCpu);
12952	}
12953	}
12954
12955	#ifdef VBOX_STRICT
12956	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12957	Assert(!fIOWrite);
12958	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12959	Assert(fIOWrite);
12960	else
12961	{
12962	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12963	* statuses, that the VMM device and some others may return. See
12964	* IOM_SUCCESS() for guidance. */
12965	AssertMsg( RT_FAILURE(rcStrict)
12966	\|\| rcStrict == VINF_SUCCESS
12967	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12968	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12969	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12970	\|\| rcStrict == VINF_EM_RAW_TO_R3
12971	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12972	#endif
12973	}
12974	#endif
12975
12976	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12977	return rcStrict;
12978	}
12979
12980
12981	/**
12982	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12983	* VM-exit.
12984	*/
12985	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12986	{
12987	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12988
12989	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12990	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12991	AssertRCReturn(rc, rc);
12992	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12993	{
12994	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12995	AssertRCReturn(rc, rc);
12996	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12997	{
12998	uint32_t uErrCode;
12999	RTGCUINTPTR GCPtrFaultAddress;
13000	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13001	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13002	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
13003	if (fErrorCodeValid)
13004	{
13005	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13006	AssertRCReturn(rc, rc);
13007	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
13008	}
13009	else
13010	uErrCode = 0;
13011
13012	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13013	&& uVector == X86_XCPT_PF)
13014	GCPtrFaultAddress = pMixedCtx->cr2;
13015	else
13016	GCPtrFaultAddress = 0;
13017
13018	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13019	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
13020
13021	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
13022	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13023	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13024	}
13025	}
13026
13027	/* Fall back to the interpreter to emulate the task-switch. */
13028	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13029	return VERR_EM_INTERPRETER;
13030	}
13031
13032
13033	/**
13034	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
13035	*/
13036	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13037	{
13038	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13039	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
13040	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
13041	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13042	AssertRCReturn(rc, rc);
13043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
13044	return VINF_EM_DBG_STEPPED;
13045	}
13046
13047
13048	/**
13049	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
13050	*/
13051	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13052	{
13053	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13054
13055	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
13056
13057	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13058	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13059	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13060	{
13061	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
13062	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13063	{
13064	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13065	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13066	}
13067	}
13068	else
13069	{
13070	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13071	rcStrict1 = VINF_SUCCESS;
13072	return rcStrict1;
13073	}
13074
13075	#if 0
13076	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13077	* just sync the whole thing. */
13078	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13079	#else
13080	/* Aggressive state sync. for now. */
13081	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13082	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13083	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13084	#endif
13085	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13086	AssertRCReturn(rc, rc);
13087
13088	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13089	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13090	VBOXSTRICTRC rcStrict2;
13091	switch (uAccessType)
13092	{
13093	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13094	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13095	{
13096	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13097	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13098	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13099
13100	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13101	GCPhys &= PAGE_BASE_GC_MASK;
13102	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13103	PVM pVM = pVCpu->CTX_SUFF(pVM);
13104	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13105	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13106
13107	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13108	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13109	CPUMCTX2CORE(pMixedCtx), GCPhys);
13110	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13111	if ( rcStrict2 == VINF_SUCCESS
13112	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13113	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13114	{
13115	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13116	\| HM_CHANGED_GUEST_RSP
13117	\| HM_CHANGED_GUEST_RFLAGS
13118	\| HM_CHANGED_GUEST_APIC_STATE);
13119	rcStrict2 = VINF_SUCCESS;
13120	}
13121	break;
13122	}
13123
13124	default:
13125	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13126	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13127	break;
13128	}
13129
13130	if (rcStrict2 != VINF_SUCCESS)
13131	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13132	return rcStrict2;
13133	}
13134
13135
13136	/**
13137	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13138	* VM-exit.
13139	*/
13140	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13141	{
13142	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13143
13144	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13145	if (pVmxTransient->fWasGuestDebugStateActive)
13146	{
13147	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13148	HMVMX_RETURN_UNEXPECTED_EXIT();
13149	}
13150
13151	if ( !pVCpu->hm.s.fSingleInstruction
13152	&& !pVmxTransient->fWasHyperDebugStateActive)
13153	{
13154	Assert(!DBGFIsStepping(pVCpu));
13155	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13156
13157	/* Don't intercept MOV DRx any more. */
13158	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13159	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13160	AssertRCReturn(rc, rc);
13161
13162	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13163	VMMRZCallRing3Disable(pVCpu);
13164	HM_DISABLE_PREEMPT();
13165
13166	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13167	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13168	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13169
13170	HM_RESTORE_PREEMPT();
13171	VMMRZCallRing3Enable(pVCpu);
13172
13173	#ifdef VBOX_WITH_STATISTICS
13174	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13175	AssertRCReturn(rc, rc);
13176	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13177	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13178	else
13179	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13180	#endif
13181	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13182	return VINF_SUCCESS;
13183	}
13184
13185	/*
13186	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13187	* Update the segment registers and DR7 from the CPU.
13188	*/
13189	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13190	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13191	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13192	AssertRCReturn(rc, rc);
13193	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13194
13195	PVM pVM = pVCpu->CTX_SUFF(pVM);
13196	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13197	{
13198	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13199	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13200	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13201	if (RT_SUCCESS(rc))
13202	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13203	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13204	}
13205	else
13206	{
13207	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13208	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13209	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13210	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13211	}
13212
13213	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13214	if (RT_SUCCESS(rc))
13215	{
13216	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13217	AssertRCReturn(rc2, rc2);
13218	return VINF_SUCCESS;
13219	}
13220	return rc;
13221	}
13222
13223
13224	/**
13225	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13226	* Conditional VM-exit.
13227	*/
13228	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13229	{
13230	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13231	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13232
13233	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13234	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13235	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13236	{
13237	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13238	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13239	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13240	{
13241	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13242	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13243	}
13244	}
13245	else
13246	{
13247	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13248	rcStrict1 = VINF_SUCCESS;
13249	return rcStrict1;
13250	}
13251
13252	RTGCPHYS GCPhys = 0;
13253	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13254
13255	#if 0
13256	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13257	#else
13258	/* Aggressive state sync. for now. */
13259	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13260	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13261	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13262	#endif
13263	AssertRCReturn(rc, rc);
13264
13265	/*
13266	* If we succeed, resume guest execution.
13267	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13268	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13269	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13270	* weird case. See @bugref{6043}.
13271	*/
13272	PVM pVM = pVCpu->CTX_SUFF(pVM);
13273	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13274	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13275	if ( rcStrict2 == VINF_SUCCESS
13276	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13277	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13278	{
13279	/* Successfully handled MMIO operation. */
13280	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13281	\| HM_CHANGED_GUEST_RSP
13282	\| HM_CHANGED_GUEST_RFLAGS
13283	\| HM_CHANGED_GUEST_APIC_STATE);
13284	return VINF_SUCCESS;
13285	}
13286	return rcStrict2;
13287	}
13288
13289
13290	/**
13291	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13292	* VM-exit.
13293	*/
13294	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13295	{
13296	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13297	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13298
13299	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13300	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13301	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13302	{
13303	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13304	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13305	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13306	}
13307	else
13308	{
13309	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13310	rcStrict1 = VINF_SUCCESS;
13311	return rcStrict1;
13312	}
13313
13314	RTGCPHYS GCPhys = 0;
13315	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13316	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13317	#if 0
13318	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13319	#else
13320	/* Aggressive state sync. for now. */
13321	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13322	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13323	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13324	#endif
13325	AssertRCReturn(rc, rc);
13326
13327	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13328	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13329
13330	RTGCUINT uErrorCode = 0;
13331	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13332	uErrorCode \|= X86_TRAP_PF_ID;
13333	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13334	uErrorCode \|= X86_TRAP_PF_RW;
13335	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13336	uErrorCode \|= X86_TRAP_PF_P;
13337
13338	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13339
13340	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13341	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13342
13343	/* Handle the pagefault trap for the nested shadow table. */
13344	PVM pVM = pVCpu->CTX_SUFF(pVM);
13345	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13346	TRPMResetTrap(pVCpu);
13347
13348	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13349	if ( rcStrict2 == VINF_SUCCESS
13350	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13351	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13352	{
13353	/* Successfully synced our nested page tables. */
13354	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13355	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13356	\| HM_CHANGED_GUEST_RSP
13357	\| HM_CHANGED_GUEST_RFLAGS);
13358	return VINF_SUCCESS;
13359	}
13360
13361	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13362	return rcStrict2;
13363	}
13364
13365	/** @} */
13366
13367	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13368	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13369	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13370
13371	/** @name VM-exit exception handlers.
13372	* @{
13373	*/
13374
13375	/**
13376	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13377	*/
13378	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13379	{
13380	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13382
13383	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13384	AssertRCReturn(rc, rc);
13385
13386	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13387	{
13388	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13389	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13390
13391	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13392	* provides VM-exit instruction length. If this causes problem later,
13393	* disassemble the instruction like it's done on AMD-V. */
13394	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13395	AssertRCReturn(rc2, rc2);
13396	return rc;
13397	}
13398
13399	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13400	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13401	return rc;
13402	}
13403
13404
13405	/**
13406	* VM-exit exception handler for \#BP (Breakpoint exception).
13407	*/
13408	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13409	{
13410	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13411	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13412
13413	/** @todo Try optimize this by not saving the entire guest state unless
13414	* really needed. */
13415	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13416	AssertRCReturn(rc, rc);
13417
13418	PVM pVM = pVCpu->CTX_SUFF(pVM);
13419	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13420	if (rc == VINF_EM_RAW_GUEST_TRAP)
13421	{
13422	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13423	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13424	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13425	AssertRCReturn(rc, rc);
13426
13427	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13428	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13429	}
13430
13431	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13432	return rc;
13433	}
13434
13435
13436	/**
13437	* VM-exit exception handler for \#AC (alignment check exception).
13438	*/
13439	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13440	{
13441	RT_NOREF_PV(pMixedCtx);
13442	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13443
13444	/*
13445	* Re-inject it. We'll detect any nesting before getting here.
13446	*/
13447	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13448	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13449	AssertRCReturn(rc, rc);
13450	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13451
13452	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13453	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13454	return VINF_SUCCESS;
13455	}
13456
13457
13458	/**
13459	* VM-exit exception handler for \#DB (Debug exception).
13460	*/
13461	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13462	{
13463	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13464	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13465	Log6(("XcptDB\n"));
13466
13467	/*
13468	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13469	* for processing.
13470	*/
13471	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13472	AssertRCReturn(rc, rc);
13473
13474	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13475	uint64_t uDR6 = X86_DR6_INIT_VAL;
13476	uDR6 \|= ( pVmxTransient->uExitQualification
13477	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13478
13479	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13480	if (rc == VINF_EM_RAW_GUEST_TRAP)
13481	{
13482	/*
13483	* The exception was for the guest. Update DR6, DR7.GD and
13484	* IA32_DEBUGCTL.LBR before forwarding it.
13485	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13486	*/
13487	VMMRZCallRing3Disable(pVCpu);
13488	HM_DISABLE_PREEMPT();
13489
13490	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13491	pMixedCtx->dr[6] \|= uDR6;
13492	if (CPUMIsGuestDebugStateActive(pVCpu))
13493	ASMSetDR6(pMixedCtx->dr[6]);
13494
13495	HM_RESTORE_PREEMPT();
13496	VMMRZCallRing3Enable(pVCpu);
13497
13498	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13499	AssertRCReturn(rc, rc);
13500
13501	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13502	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13503
13504	/* Paranoia. */
13505	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13506	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13507
13508	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13509	AssertRCReturn(rc, rc);
13510
13511	/*
13512	* Raise #DB in the guest.
13513	*
13514	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13515	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP (INT1) and not the
13516	* regular #DB. Thus it -may- trigger different handling in the CPU (like skipped DPL checks), see @bugref{6398}.
13517	*
13518	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of Intel 386,
13519	* see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13520	*/
13521	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13522	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13523	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13524	AssertRCReturn(rc, rc);
13525	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13526	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13527	return VINF_SUCCESS;
13528	}
13529
13530	/*
13531	* Not a guest trap, must be a hypervisor related debug event then.
13532	* Update DR6 in case someone is interested in it.
13533	*/
13534	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13535	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13536	CPUMSetHyperDR6(pVCpu, uDR6);
13537
13538	return rc;
13539	}
13540
13541
13542	/**
13543	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13544	* point exception).
13545	*/
13546	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13547	{
13548	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13549
13550	/* We require CR0 and EFER. EFER is always up-to-date. */
13551	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13552	AssertRCReturn(rc, rc);
13553
13554	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13555	VMMRZCallRing3Disable(pVCpu);
13556	HM_DISABLE_PREEMPT();
13557
13558	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13559	if (pVmxTransient->fWasGuestFPUStateActive)
13560	{
13561	rc = VINF_EM_RAW_GUEST_TRAP;
13562	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13563	}
13564	else
13565	{
13566	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13567	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13568	#endif
13569	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13570	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13571	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13572	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13573	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13574	}
13575
13576	HM_RESTORE_PREEMPT();
13577	VMMRZCallRing3Enable(pVCpu);
13578
13579	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13580	{
13581	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13582	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13583	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13584	pVCpu->hm.s.fPreloadGuestFpu = true;
13585	}
13586	else
13587	{
13588	/* Forward #NM to the guest. */
13589	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13590	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13591	AssertRCReturn(rc, rc);
13592	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13593	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13595	}
13596
13597	return VINF_SUCCESS;
13598	}
13599
13600
13601	/**
13602	* VM-exit exception handler for \#GP (General-protection exception).
13603	*
13604	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13605	*/
13606	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13607	{
13608	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13610
13611	int rc;
13612	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13613	{ /* likely */ }
13614	else
13615	{
13616	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13617	Assert(pVCpu->hm.s.fUsingDebugLoop);
13618	#endif
13619	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13620	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13621	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13622	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13623	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13624	AssertRCReturn(rc, rc);
13625	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13626	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13627	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13628	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13629	return rc;
13630	}
13631
13632	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13633	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13634
13635	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13636	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13637	AssertRCReturn(rc, rc);
13638
13639	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13640	uint32_t cbOp = 0;
13641	PVM pVM = pVCpu->CTX_SUFF(pVM);
13642	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13643	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13644	if (RT_SUCCESS(rc))
13645	{
13646	rc = VINF_SUCCESS;
13647	Assert(cbOp == pDis->cbInstr);
13648	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13649	switch (pDis->pCurInstr->uOpcode)
13650	{
13651	case OP_CLI:
13652	{
13653	pMixedCtx->eflags.Bits.u1IF = 0;
13654	pMixedCtx->eflags.Bits.u1RF = 0;
13655	pMixedCtx->rip += pDis->cbInstr;
13656	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13657	if ( !fDbgStepping
13658	&& pMixedCtx->eflags.Bits.u1TF)
13659	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13660	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13661	break;
13662	}
13663
13664	case OP_STI:
13665	{
13666	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13667	pMixedCtx->eflags.Bits.u1IF = 1;
13668	pMixedCtx->eflags.Bits.u1RF = 0;
13669	pMixedCtx->rip += pDis->cbInstr;
13670	if (!fOldIF)
13671	{
13672	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13673	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13674	}
13675	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13676	if ( !fDbgStepping
13677	&& pMixedCtx->eflags.Bits.u1TF)
13678	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13679	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13680	break;
13681	}
13682
13683	case OP_HLT:
13684	{
13685	rc = VINF_EM_HALT;
13686	pMixedCtx->rip += pDis->cbInstr;
13687	pMixedCtx->eflags.Bits.u1RF = 0;
13688	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13689	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13690	break;
13691	}
13692
13693	case OP_POPF:
13694	{
13695	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13696	uint32_t cbParm;
13697	uint32_t uMask;
13698	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13699	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13700	{
13701	cbParm = 4;
13702	uMask = 0xffffffff;
13703	}
13704	else
13705	{
13706	cbParm = 2;
13707	uMask = 0xffff;
13708	}
13709
13710	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13711	RTGCPTR GCPtrStack = 0;
13712	X86EFLAGS Eflags;
13713	Eflags.u32 = 0;
13714	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13715	&GCPtrStack);
13716	if (RT_SUCCESS(rc))
13717	{
13718	Assert(sizeof(Eflags.u32) >= cbParm);
13719	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13720	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13721	}
13722	if (RT_FAILURE(rc))
13723	{
13724	rc = VERR_EM_INTERPRETER;
13725	break;
13726	}
13727	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13728	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13729	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13730	pMixedCtx->esp += cbParm;
13731	pMixedCtx->esp &= uMask;
13732	pMixedCtx->rip += pDis->cbInstr;
13733	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13734	\| HM_CHANGED_GUEST_RSP
13735	\| HM_CHANGED_GUEST_RFLAGS);
13736	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13737	POPF restores EFLAGS.TF. */
13738	if ( !fDbgStepping
13739	&& fGstStepping)
13740	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13741	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13742	break;
13743	}
13744
13745	case OP_PUSHF:
13746	{
13747	uint32_t cbParm;
13748	uint32_t uMask;
13749	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13750	{
13751	cbParm = 4;
13752	uMask = 0xffffffff;
13753	}
13754	else
13755	{
13756	cbParm = 2;
13757	uMask = 0xffff;
13758	}
13759
13760	/* Get the stack pointer & push the contents of eflags onto the stack. */
13761	RTGCPTR GCPtrStack = 0;
13762	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13763	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13764	if (RT_FAILURE(rc))
13765	{
13766	rc = VERR_EM_INTERPRETER;
13767	break;
13768	}
13769	X86EFLAGS Eflags = pMixedCtx->eflags;
13770	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13771	Eflags.Bits.u1RF = 0;
13772	Eflags.Bits.u1VM = 0;
13773
13774	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13775	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13776	{
13777	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13778	rc = VERR_EM_INTERPRETER;
13779	break;
13780	}
13781	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13782	pMixedCtx->esp -= cbParm;
13783	pMixedCtx->esp &= uMask;
13784	pMixedCtx->rip += pDis->cbInstr;
13785	pMixedCtx->eflags.Bits.u1RF = 0;
13786	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13787	\| HM_CHANGED_GUEST_RSP
13788	\| HM_CHANGED_GUEST_RFLAGS);
13789	if ( !fDbgStepping
13790	&& pMixedCtx->eflags.Bits.u1TF)
13791	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13792	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13793	break;
13794	}
13795
13796	case OP_IRET:
13797	{
13798	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13799	* instruction reference. */
13800	RTGCPTR GCPtrStack = 0;
13801	uint32_t uMask = 0xffff;
13802	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13803	uint16_t aIretFrame[3];
13804	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13805	{
13806	rc = VERR_EM_INTERPRETER;
13807	break;
13808	}
13809	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13810	&GCPtrStack);
13811	if (RT_SUCCESS(rc))
13812	{
13813	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13814	PGMACCESSORIGIN_HM));
13815	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13816	}
13817	if (RT_FAILURE(rc))
13818	{
13819	rc = VERR_EM_INTERPRETER;
13820	break;
13821	}
13822	pMixedCtx->eip = 0;
13823	pMixedCtx->ip = aIretFrame[0];
13824	pMixedCtx->cs.Sel = aIretFrame[1];
13825	pMixedCtx->cs.ValidSel = aIretFrame[1];
13826	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13827	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13828	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13829	pMixedCtx->sp += sizeof(aIretFrame);
13830	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13831	\| HM_CHANGED_GUEST_SEGMENT_REGS
13832	\| HM_CHANGED_GUEST_RSP
13833	\| HM_CHANGED_GUEST_RFLAGS);
13834	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13835	if ( !fDbgStepping
13836	&& fGstStepping)
13837	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13838	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13839	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13840	break;
13841	}
13842
13843	case OP_INT:
13844	{
13845	uint16_t uVector = pDis->Param1.uValue & 0xff;
13846	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13847	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13848	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13849	break;
13850	}
13851
13852	case OP_INTO:
13853	{
13854	if (pMixedCtx->eflags.Bits.u1OF)
13855	{
13856	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13857	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13858	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13859	}
13860	else
13861	{
13862	pMixedCtx->eflags.Bits.u1RF = 0;
13863	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13864	}
13865	break;
13866	}
13867
13868	default:
13869	{
13870	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13871	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13872	EMCODETYPE_SUPERVISOR);
13873	rc = VBOXSTRICTRC_VAL(rc2);
13874	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13875	/** @todo We have to set pending-debug exceptions here when the guest is
13876	* single-stepping depending on the instruction that was interpreted. */
13877	Log4(("#GP rc=%Rrc\n", rc));
13878	break;
13879	}
13880	}
13881	}
13882	else
13883	rc = VERR_EM_INTERPRETER;
13884
13885	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13886	("#GP Unexpected rc=%Rrc\n", rc));
13887	return rc;
13888	}
13889
13890
13891	/**
13892	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13893	* the exception reported in the VMX transient structure back into the VM.
13894	*
13895	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13896	* up-to-date.
13897	*/
13898	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13899	{
13900	RT_NOREF_PV(pMixedCtx);
13901	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13902	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13903	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13904	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13905	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13906	#endif
13907
13908	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13909	hmR0VmxCheckExitDueToEventDelivery(). */
13910	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13911	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13912	AssertRCReturn(rc, rc);
13913	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13914
13915	#ifdef DEBUG_ramshankar
13916	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13917	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13918	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13919	#endif
13920
13921	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13922	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13923	return VINF_SUCCESS;
13924	}
13925
13926
13927	/**
13928	* VM-exit exception handler for \#PF (Page-fault exception).
13929	*/
13930	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13931	{
13932	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13933	PVM pVM = pVCpu->CTX_SUFF(pVM);
13934	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13935	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13936	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13937	AssertRCReturn(rc, rc);
13938
13939	if (!pVM->hm.s.fNestedPaging)
13940	{ /* likely */ }
13941	else
13942	{
13943	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13944	Assert(pVCpu->hm.s.fUsingDebugLoop);
13945	#endif
13946	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13947	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13948	{
13949	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13950	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13951	}
13952	else
13953	{
13954	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13955	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13956	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13957	}
13958	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13959	return rc;
13960	}
13961
13962	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13963	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13964	if (pVmxTransient->fVectoringPF)
13965	{
13966	Assert(pVCpu->hm.s.Event.fPending);
13967	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13968	}
13969
13970	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13971	AssertRCReturn(rc, rc);
13972
13973	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13974	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13975
13976	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13977	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13978	(RTGCPTR)pVmxTransient->uExitQualification);
13979
13980	Log4(("#PF: rc=%Rrc\n", rc));
13981	if (rc == VINF_SUCCESS)
13982	{
13983	#if 0
13984	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13985	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13986	* memory? We don't update the whole state here... */
13987	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13988	\| HM_CHANGED_GUEST_RSP
13989	\| HM_CHANGED_GUEST_RFLAGS
13990	\| HM_CHANGED_GUEST_APIC_STATE);
13991	#else
13992	/*
13993	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13994	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13995	*/
13996	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13997	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13998	#endif
13999	TRPMResetTrap(pVCpu);
14000	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
14001	return rc;
14002	}
14003
14004	if (rc == VINF_EM_RAW_GUEST_TRAP)
14005	{
14006	if (!pVmxTransient->fVectoringDoublePF)
14007	{
14008	/* It's a guest page fault and needs to be reflected to the guest. */
14009	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
14010	TRPMResetTrap(pVCpu);
14011	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
14012	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14013	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
14014	}
14015	else
14016	{
14017	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14018	TRPMResetTrap(pVCpu);
14019	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14020	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
14021	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
14022	}
14023
14024	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14025	return VINF_SUCCESS;
14026	}
14027
14028	TRPMResetTrap(pVCpu);
14029	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14030	return rc;
14031	}
14032
14033	/** @} */
14034

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72390

Download in other formats: