HMVMXR0.cpp@ 71881

Last change on this file since 71881 was 71415, checked in by vboxsync, 7 years ago
VMM/HM: Get rid of lazy FPU loading for AMD-V. Nested Hw.virt: Fix FPU related issues while executing nested-KVM DSL guests.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 600.3 KB

Line
1	/* $Id: HMVMXR0.cpp 71415 2018-03-21 09:29:22Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#define HMVMX_USE_IEM_EVENT_REFLECTION
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50	# define HMVMX_ALWAYS_TRAP_PF
51	# define HMVMX_ALWAYS_SWAP_FPU_STATE
52	# define HMVMX_ALWAYS_FLUSH_TLB
53	# define HMVMX_ALWAYS_SWAP_EFER
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DR7 \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_INTR_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Exception bitmap mask for real-mode guests (real-on-v86).
140	*
141	* We need to intercept all exceptions manually except:
142	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144	* due to bugs in Intel CPUs.
145	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
146	* support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*********************************************************************************************************************************
197	* Structures and Typedefs *
198	*********************************************************************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG fEFlags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit code qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u7Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	/** INVEPT, INVVPID, INVPCID information. */
247	struct
248	{
249	/** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
250	uint32_t u2Scaling : 2;
251	uint32_t u5Reserved0 : 5;
252	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253	uint32_t u3AddrSize : 3;
254	uint32_t u1Reserved0 : 1;
255	uint32_t u4Reserved0 : 4;
256	/** The segment register (X86_SREG_XXX). */
257	uint32_t iSegReg : 3;
258	/** The index register (X86_GREG_XXX). */
259	uint32_t iIdxReg : 4;
260	/** Set if index register is invalid. */
261	uint32_t fIdxRegValid : 1;
262	/** The base register (X86_GREG_XXX). */
263	uint32_t iBaseReg : 4;
264	/** Set if base register is invalid. */
265	uint32_t fBaseRegValid : 1;
266	/** Register 2 (X86_GREG_XXX). */
267	uint32_t iReg2 : 4;
268	} Inv;
269	} ExitInstrInfo;
270	/** Whether the VM-entry failed or not. */
271	bool fVMEntryFailed;
272	/** Alignment. */
273	uint8_t abAlignment1[3];
274
275	/** The VM-entry interruption-information field. */
276	uint32_t uEntryIntInfo;
277	/** The VM-entry exception error code field. */
278	uint32_t uEntryXcptErrorCode;
279	/** The VM-entry instruction length field. */
280	uint32_t cbEntryInstr;
281
282	/** IDT-vectoring information field. */
283	uint32_t uIdtVectoringInfo;
284	/** IDT-vectoring error code. */
285	uint32_t uIdtVectoringErrorCode;
286
287	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
288	uint32_t fVmcsFieldsRead;
289
290	/** Whether the guest FPU was active at the time of VM-exit. */
291	bool fWasGuestFPUStateActive;
292	/** Whether the guest debug state was active at the time of VM-exit. */
293	bool fWasGuestDebugStateActive;
294	/** Whether the hyper debug state was active at the time of VM-exit. */
295	bool fWasHyperDebugStateActive;
296	/** Whether TSC-offsetting should be setup before VM-entry. */
297	bool fUpdateTscOffsettingAndPreemptTimer;
298	/** Whether the VM-exit was caused by a page-fault during delivery of a
299	* contributory exception or a page-fault. */
300	bool fVectoringDoublePF;
301	/** Whether the VM-exit was caused by a page-fault during delivery of an
302	* external interrupt or NMI. */
303	bool fVectoringPF;
304	} VMXTRANSIENT;
305	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
306	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
307	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
308	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
309	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
310	/** Pointer to VMX transient state. */
311	typedef VMXTRANSIENT *PVMXTRANSIENT;
312
313
314	/**
315	* MSR-bitmap read permissions.
316	*/
317	typedef enum VMXMSREXITREAD
318	{
319	/** Reading this MSR causes a VM-exit. */
320	VMXMSREXIT_INTERCEPT_READ = 0xb,
321	/** Reading this MSR does not cause a VM-exit. */
322	VMXMSREXIT_PASSTHRU_READ
323	} VMXMSREXITREAD;
324	/** Pointer to MSR-bitmap read permissions. */
325	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
326
327	/**
328	* MSR-bitmap write permissions.
329	*/
330	typedef enum VMXMSREXITWRITE
331	{
332	/** Writing to this MSR causes a VM-exit. */
333	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
334	/** Writing to this MSR does not cause a VM-exit. */
335	VMXMSREXIT_PASSTHRU_WRITE
336	} VMXMSREXITWRITE;
337	/** Pointer to MSR-bitmap write permissions. */
338	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
339
340
341	/**
342	* VMX VM-exit handler.
343	*
344	* @returns Strict VBox status code (i.e. informational status codes too).
345	* @param pVCpu The cross context virtual CPU structure.
346	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
347	* out-of-sync. Make sure to update the required
348	* fields before using them.
349	* @param pVmxTransient Pointer to the VMX-transient structure.
350	*/
351	#ifndef HMVMX_USE_FUNCTION_TABLE
352	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353	#else
354	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355	/** Pointer to VM-exit handler. */
356	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
357	#endif
358
359	/**
360	* VMX VM-exit handler, non-strict status code.
361	*
362	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
363	*
364	* @returns VBox status code, no informational status code returned.
365	* @param pVCpu The cross context virtual CPU structure.
366	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
367	* out-of-sync. Make sure to update the required
368	* fields before using them.
369	* @param pVmxTransient Pointer to the VMX-transient structure.
370	*
371	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
372	* use of that status code will be replaced with VINF_EM_SOMETHING
373	* later when switching over to IEM.
374	*/
375	#ifndef HMVMX_USE_FUNCTION_TABLE
376	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
377	#else
378	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
379	#endif
380
381
382	/*********************************************************************************************************************************
383	* Internal Functions *
384	*********************************************************************************************************************************/
385	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
386	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
387	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
388	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
389	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
390	bool fStepping, uint32_t *puIntState);
391	#if HC_ARCH_BITS == 32
392	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
393	#endif
394	#ifndef HMVMX_USE_FUNCTION_TABLE
395	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
396	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
397	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
398	#else
399	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
400	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
401	#endif
402
403
404	/** @name VM-exit handlers.
405	* @{
406	*/
407	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
408	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
409	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
415	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
416	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
417	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
418	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
419	static FNVMXEXITHANDLER hmR0VmxExitHlt;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
421	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
422	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
423	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
424	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
425	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
426	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
427	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
428	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
429	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
430	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
431	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
432	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
433	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
434	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
435	static FNVMXEXITHANDLER hmR0VmxExitMwait;
436	static FNVMXEXITHANDLER hmR0VmxExitMtf;
437	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
438	static FNVMXEXITHANDLER hmR0VmxExitPause;
439	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
440	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
441	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
442	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
443	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
444	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
445	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
446	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
447	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
448	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
449	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
450	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
451	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
452	/** @} */
453
454	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
458	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
459	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
460	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
461	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
462	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
463
464
465	/*********************************************************************************************************************************
466	* Global Variables *
467	*********************************************************************************************************************************/
468	#ifdef HMVMX_USE_FUNCTION_TABLE
469
470	/**
471	* VMX_EXIT dispatch table.
472	*/
473	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
474	{
475	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
476	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
477	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
478	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
479	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
480	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
481	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
482	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
483	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
484	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
485	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
486	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
487	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
488	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
489	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
490	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
491	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
492	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
493	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
494	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
495	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
496	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
497	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
498	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
499	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
500	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
501	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
502	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
503	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
504	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
505	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
506	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
507	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
508	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
509	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
510	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
511	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
512	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
513	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
514	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
515	/* 40 UNDEFINED */ hmR0VmxExitPause,
516	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
517	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
518	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
519	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
520	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
521	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
522	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
523	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
524	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
525	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
526	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
527	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
528	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
529	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
530	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
531	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
532	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
533	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
534	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
535	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
536	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
537	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
538	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
539	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
540	};
541	#endif /* HMVMX_USE_FUNCTION_TABLE */
542
543	#ifdef VBOX_STRICT
544	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
545	{
546	/* 0 */ "(Not Used)",
547	/* 1 */ "VMCALL executed in VMX root operation.",
548	/* 2 */ "VMCLEAR with invalid physical address.",
549	/* 3 */ "VMCLEAR with VMXON pointer.",
550	/* 4 */ "VMLAUNCH with non-clear VMCS.",
551	/* 5 */ "VMRESUME with non-launched VMCS.",
552	/* 6 */ "VMRESUME after VMXOFF",
553	/* 7 */ "VM-entry with invalid control fields.",
554	/* 8 */ "VM-entry with invalid host state fields.",
555	/* 9 */ "VMPTRLD with invalid physical address.",
556	/* 10 */ "VMPTRLD with VMXON pointer.",
557	/* 11 */ "VMPTRLD with incorrect revision identifier.",
558	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
559	/* 13 */ "VMWRITE to read-only VMCS component.",
560	/* 14 */ "(Not Used)",
561	/* 15 */ "VMXON executed in VMX root operation.",
562	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
563	/* 17 */ "VM-entry with non-launched executing VMCS.",
564	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
565	/* 19 */ "VMCALL with non-clear VMCS.",
566	/* 20 */ "VMCALL with invalid VM-exit control fields.",
567	/* 21 */ "(Not Used)",
568	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
569	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
570	/* 24 */ "VMCALL with invalid SMM-monitor features.",
571	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
572	/* 26 */ "VM-entry with events blocked by MOV SS.",
573	/* 27 */ "(Not Used)",
574	/* 28 */ "Invalid operand to INVEPT/INVVPID."
575	};
576	#endif /* VBOX_STRICT */
577
578
579
580	/**
581	* Updates the VM's last error record.
582	*
583	* If there was a VMX instruction error, reads the error data from the VMCS and
584	* updates VCPU's last error record as well.
585	*
586	* @param pVM The cross context VM structure.
587	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
588	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
589	* VERR_VMX_INVALID_VMCS_FIELD.
590	* @param rc The error code.
591	*/
592	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
593	{
594	AssertPtr(pVM);
595	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
596	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
597	{
598	AssertPtrReturnVoid(pVCpu);
599	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
600	}
601	pVM->hm.s.lLastError = rc;
602	}
603
604
605	/**
606	* Reads the VM-entry interruption-information field from the VMCS into the VMX
607	* transient structure.
608	*
609	* @returns VBox status code.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*
612	* @remarks No-long-jump zone!!!
613	*/
614	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
615	{
616	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
617	AssertRCReturn(rc, rc);
618	return VINF_SUCCESS;
619	}
620
621
622	#ifdef VBOX_STRICT
623	/**
624	* Reads the VM-entry exception error code field from the VMCS into
625	* the VMX transient structure.
626	*
627	* @returns VBox status code.
628	* @param pVmxTransient Pointer to the VMX transient structure.
629	*
630	* @remarks No-long-jump zone!!!
631	*/
632	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
633	{
634	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
635	AssertRCReturn(rc, rc);
636	return VINF_SUCCESS;
637	}
638	#endif /* VBOX_STRICT */
639
640
641	#ifdef VBOX_STRICT
642	/**
643	* Reads the VM-entry exception error code field from the VMCS into
644	* the VMX transient structure.
645	*
646	* @returns VBox status code.
647	* @param pVmxTransient Pointer to the VMX transient structure.
648	*
649	* @remarks No-long-jump zone!!!
650	*/
651	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
652	{
653	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
654	AssertRCReturn(rc, rc);
655	return VINF_SUCCESS;
656	}
657	#endif /* VBOX_STRICT */
658
659
660	/**
661	* Reads the VM-exit interruption-information field from the VMCS into the VMX
662	* transient structure.
663	*
664	* @returns VBox status code.
665	* @param pVmxTransient Pointer to the VMX transient structure.
666	*/
667	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
668	{
669	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
670	{
671	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
672	AssertRCReturn(rc, rc);
673	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
674	}
675	return VINF_SUCCESS;
676	}
677
678
679	/**
680	* Reads the VM-exit interruption error code from the VMCS into the VMX
681	* transient structure.
682	*
683	* @returns VBox status code.
684	* @param pVmxTransient Pointer to the VMX transient structure.
685	*/
686	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
687	{
688	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
689	{
690	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
691	AssertRCReturn(rc, rc);
692	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
693	}
694	return VINF_SUCCESS;
695	}
696
697
698	/**
699	* Reads the VM-exit instruction length field from the VMCS into the VMX
700	* transient structure.
701	*
702	* @returns VBox status code.
703	* @param pVmxTransient Pointer to the VMX transient structure.
704	*/
705	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
706	{
707	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
708	{
709	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
710	AssertRCReturn(rc, rc);
711	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
712	}
713	return VINF_SUCCESS;
714	}
715
716
717	/**
718	* Reads the VM-exit instruction-information field from the VMCS into
719	* the VMX transient structure.
720	*
721	* @returns VBox status code.
722	* @param pVmxTransient Pointer to the VMX transient structure.
723	*/
724	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
725	{
726	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
727	{
728	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
729	AssertRCReturn(rc, rc);
730	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
731	}
732	return VINF_SUCCESS;
733	}
734
735
736	/**
737	* Reads the exit code qualification from the VMCS into the VMX transient
738	* structure.
739	*
740	* @returns VBox status code.
741	* @param pVCpu The cross context virtual CPU structure of the
742	* calling EMT. (Required for the VMCS cache case.)
743	* @param pVmxTransient Pointer to the VMX transient structure.
744	*/
745	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
746	{
747	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
748	{
749	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
750	AssertRCReturn(rc, rc);
751	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
752	}
753	return VINF_SUCCESS;
754	}
755
756
757	/**
758	* Reads the IDT-vectoring information field from the VMCS into the VMX
759	* transient structure.
760	*
761	* @returns VBox status code.
762	* @param pVmxTransient Pointer to the VMX transient structure.
763	*
764	* @remarks No-long-jump zone!!!
765	*/
766	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
767	{
768	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
769	{
770	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
771	AssertRCReturn(rc, rc);
772	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
773	}
774	return VINF_SUCCESS;
775	}
776
777
778	/**
779	* Reads the IDT-vectoring error code from the VMCS into the VMX
780	* transient structure.
781	*
782	* @returns VBox status code.
783	* @param pVmxTransient Pointer to the VMX transient structure.
784	*/
785	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
786	{
787	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
788	{
789	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
790	AssertRCReturn(rc, rc);
791	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
792	}
793	return VINF_SUCCESS;
794	}
795
796
797	/**
798	* Enters VMX root mode operation on the current CPU.
799	*
800	* @returns VBox status code.
801	* @param pVM The cross context VM structure. Can be
802	* NULL, after a resume.
803	* @param HCPhysCpuPage Physical address of the VMXON region.
804	* @param pvCpuPage Pointer to the VMXON region.
805	*/
806	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
807	{
808	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
809	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
810	Assert(pvCpuPage);
811	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
812
813	if (pVM)
814	{
815	/* Write the VMCS revision dword to the VMXON region. */
816	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
817	}
818
819	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
820	RTCCUINTREG fEFlags = ASMIntDisableFlags();
821
822	/* Enable the VMX bit in CR4 if necessary. */
823	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
824
825	/* Enter VMX root mode. */
826	int rc = VMXEnable(HCPhysCpuPage);
827	if (RT_FAILURE(rc))
828	{
829	if (!(uOldCr4 & X86_CR4_VMXE))
830	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
831
832	if (pVM)
833	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
834	}
835
836	/* Restore interrupts. */
837	ASMSetFlags(fEFlags);
838	return rc;
839	}
840
841
842	/**
843	* Exits VMX root mode operation on the current CPU.
844	*
845	* @returns VBox status code.
846	*/
847	static int hmR0VmxLeaveRootMode(void)
848	{
849	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
850
851	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
852	RTCCUINTREG fEFlags = ASMIntDisableFlags();
853
854	/* If we're for some reason not in VMX root mode, then don't leave it. */
855	RTCCUINTREG uHostCR4 = ASMGetCR4();
856
857	int rc;
858	if (uHostCR4 & X86_CR4_VMXE)
859	{
860	/* Exit VMX root mode and clear the VMX bit in CR4. */
861	VMXDisable();
862	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
863	rc = VINF_SUCCESS;
864	}
865	else
866	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
867
868	/* Restore interrupts. */
869	ASMSetFlags(fEFlags);
870	return rc;
871	}
872
873
874	/**
875	* Allocates and maps one physically contiguous page. The allocated page is
876	* zero'd out. (Used by various VT-x structures).
877	*
878	* @returns IPRT status code.
879	* @param pMemObj Pointer to the ring-0 memory object.
880	* @param ppVirt Where to store the virtual address of the
881	* allocation.
882	* @param pHCPhys Where to store the physical address of the
883	* allocation.
884	*/
885	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
886	{
887	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
888	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
889	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
890
891	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
892	if (RT_FAILURE(rc))
893	return rc;
894	ppVirt = RTR0MemObjAddress(pMemObj);
895	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
896	ASMMemZero32(*ppVirt, PAGE_SIZE);
897	return VINF_SUCCESS;
898	}
899
900
901	/**
902	* Frees and unmaps an allocated physical page.
903	*
904	* @param pMemObj Pointer to the ring-0 memory object.
905	* @param ppVirt Where to re-initialize the virtual address of
906	* allocation as 0.
907	* @param pHCPhys Where to re-initialize the physical address of the
908	* allocation as 0.
909	*/
910	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
911	{
912	AssertPtr(pMemObj);
913	AssertPtr(ppVirt);
914	AssertPtr(pHCPhys);
915	if (*pMemObj != NIL_RTR0MEMOBJ)
916	{
917	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
918	AssertRC(rc);
919	*pMemObj = NIL_RTR0MEMOBJ;
920	*ppVirt = 0;
921	*pHCPhys = 0;
922	}
923	}
924
925
926	/**
927	* Worker function to free VT-x related structures.
928	*
929	* @returns IPRT status code.
930	* @param pVM The cross context VM structure.
931	*/
932	static void hmR0VmxStructsFree(PVM pVM)
933	{
934	for (VMCPUID i = 0; i < pVM->cCpus; i++)
935	{
936	PVMCPU pVCpu = &pVM->aCpus[i];
937	AssertPtr(pVCpu);
938
939	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
940	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
941
942	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
944
945	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
946	}
947
948	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
949	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951	#endif
952	}
953
954
955	/**
956	* Worker function to allocate VT-x related VM structures.
957	*
958	* @returns IPRT status code.
959	* @param pVM The cross context VM structure.
960	*/
961	static int hmR0VmxStructsAlloc(PVM pVM)
962	{
963	/*
964	* Initialize members up-front so we can cleanup properly on allocation failure.
965	*/
966	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
967	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
968	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
969	pVM->hm.s.vmx.HCPhys##a_Name = 0;
970
971	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
972	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
973	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
974	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
975
976	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
977	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
978	#endif
979	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
980
981	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
982	for (VMCPUID i = 0; i < pVM->cCpus; i++)
983	{
984	PVMCPU pVCpu = &pVM->aCpus[i];
985	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
986	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
987	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
988	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
989	}
990	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
991	#undef VMXLOCAL_INIT_VM_MEMOBJ
992
993	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
994	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
995	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
996	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
997
998	/*
999	* Allocate all the VT-x structures.
1000	*/
1001	int rc = VINF_SUCCESS;
1002	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1003	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1004	if (RT_FAILURE(rc))
1005	goto cleanup;
1006	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1007	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1008	#endif
1009
1010	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1011	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1012	{
1013	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1014	&pVM->hm.s.vmx.HCPhysApicAccess);
1015	if (RT_FAILURE(rc))
1016	goto cleanup;
1017	}
1018
1019	/*
1020	* Initialize per-VCPU VT-x structures.
1021	*/
1022	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1023	{
1024	PVMCPU pVCpu = &pVM->aCpus[i];
1025	AssertPtr(pVCpu);
1026
1027	/* Allocate the VM control structure (VMCS). */
1028	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1029	if (RT_FAILURE(rc))
1030	goto cleanup;
1031
1032	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1033	if ( PDMHasApic(pVM)
1034	&& (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1035	{
1036	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1037	NULL /* pR3Ptr /, NULL / pRCPtr */);
1038	if (RT_FAILURE(rc))
1039	goto cleanup;
1040	}
1041
1042	/*
1043	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1044	* transparent accesses of specific MSRs.
1045	*
1046	* If the condition for enabling MSR bitmaps changes here, don't forget to
1047	* update HMAreMsrBitmapsAvailable().
1048	*/
1049	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1050	{
1051	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1052	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1053	if (RT_FAILURE(rc))
1054	goto cleanup;
1055	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1056	}
1057
1058	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1059	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1060	if (RT_FAILURE(rc))
1061	goto cleanup;
1062
1063	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1064	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1065	if (RT_FAILURE(rc))
1066	goto cleanup;
1067	}
1068
1069	return VINF_SUCCESS;
1070
1071	cleanup:
1072	hmR0VmxStructsFree(pVM);
1073	return rc;
1074	}
1075
1076
1077	/**
1078	* Does global VT-x initialization (called during module initialization).
1079	*
1080	* @returns VBox status code.
1081	*/
1082	VMMR0DECL(int) VMXR0GlobalInit(void)
1083	{
1084	#ifdef HMVMX_USE_FUNCTION_TABLE
1085	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1086	# ifdef VBOX_STRICT
1087	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1088	Assert(g_apfnVMExitHandlers[i]);
1089	# endif
1090	#endif
1091	return VINF_SUCCESS;
1092	}
1093
1094
1095	/**
1096	* Does global VT-x termination (called during module termination).
1097	*/
1098	VMMR0DECL(void) VMXR0GlobalTerm()
1099	{
1100	/* Nothing to do currently. */
1101	}
1102
1103
1104	/**
1105	* Sets up and activates VT-x on the current CPU.
1106	*
1107	* @returns VBox status code.
1108	* @param pCpu Pointer to the global CPU info struct.
1109	* @param pVM The cross context VM structure. Can be
1110	* NULL after a host resume operation.
1111	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1112	* fEnabledByHost is @c true).
1113	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1114	* @a fEnabledByHost is @c true).
1115	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1116	* enable VT-x on the host.
1117	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1118	*/
1119	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1120	void *pvMsrs)
1121	{
1122	Assert(pCpu);
1123	Assert(pvMsrs);
1124	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1125
1126	/* Enable VT-x if it's not already enabled by the host. */
1127	if (!fEnabledByHost)
1128	{
1129	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1130	if (RT_FAILURE(rc))
1131	return rc;
1132	}
1133
1134	/*
1135	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1136	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1137	*/
1138	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1139	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1140	{
1141	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1142	pCpu->fFlushAsidBeforeUse = false;
1143	}
1144	else
1145	pCpu->fFlushAsidBeforeUse = true;
1146
1147	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1148	++pCpu->cTlbFlushes;
1149
1150	return VINF_SUCCESS;
1151	}
1152
1153
1154	/**
1155	* Deactivates VT-x on the current CPU.
1156	*
1157	* @returns VBox status code.
1158	* @param pCpu Pointer to the global CPU info struct.
1159	* @param pvCpuPage Pointer to the VMXON region.
1160	* @param HCPhysCpuPage Physical address of the VMXON region.
1161	*
1162	* @remarks This function should never be called when SUPR0EnableVTx() or
1163	* similar was used to enable VT-x on the host.
1164	*/
1165	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1166	{
1167	NOREF(pCpu);
1168	NOREF(pvCpuPage);
1169	NOREF(HCPhysCpuPage);
1170
1171	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1172	return hmR0VmxLeaveRootMode();
1173	}
1174
1175
1176	/**
1177	* Sets the permission bits for the specified MSR in the MSR bitmap.
1178	*
1179	* @param pVCpu The cross context virtual CPU structure.
1180	* @param uMsr The MSR value.
1181	* @param enmRead Whether reading this MSR causes a VM-exit.
1182	* @param enmWrite Whether writing this MSR causes a VM-exit.
1183	*/
1184	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1185	{
1186	int32_t iBit;
1187	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189	/*
1190	* Layout:
1191	* 0x000 - 0x3ff - Low MSR read bits
1192	* 0x400 - 0x7ff - High MSR read bits
1193	* 0x800 - 0xbff - Low MSR write bits
1194	* 0xc00 - 0xfff - High MSR write bits
1195	*/
1196	if (uMsr <= 0x00001FFF)
1197	iBit = uMsr;
1198	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1199	{
1200	iBit = uMsr - UINT32_C(0xC0000000);
1201	pbMsrBitmap += 0x400;
1202	}
1203	else
1204	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1205
1206	Assert(iBit <= 0x1fff);
1207	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1208	ASMBitSet(pbMsrBitmap, iBit);
1209	else
1210	ASMBitClear(pbMsrBitmap, iBit);
1211
1212	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1213	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1214	else
1215	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1216	}
1217
1218
1219	#ifdef VBOX_STRICT
1220	/**
1221	* Gets the permission bits for the specified MSR in the MSR bitmap.
1222	*
1223	* @returns VBox status code.
1224	* @retval VINF_SUCCESS if the specified MSR is found.
1225	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1226	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1227	*
1228	* @param pVCpu The cross context virtual CPU structure.
1229	* @param uMsr The MSR.
1230	* @param penmRead Where to store the read permissions.
1231	* @param penmWrite Where to store the write permissions.
1232	*/
1233	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1234	{
1235	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1236	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1237	int32_t iBit;
1238	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1239
1240	/* See hmR0VmxSetMsrPermission() for the layout. */
1241	if (uMsr <= 0x00001FFF)
1242	iBit = uMsr;
1243	else if ( uMsr >= 0xC0000000
1244	&& uMsr <= 0xC0001FFF)
1245	{
1246	iBit = (uMsr - 0xC0000000);
1247	pbMsrBitmap += 0x400;
1248	}
1249	else
1250	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1251
1252	Assert(iBit <= 0x1fff);
1253	if (ASMBitTest(pbMsrBitmap, iBit))
1254	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1255	else
1256	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1257
1258	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1259	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1260	else
1261	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1262	return VINF_SUCCESS;
1263	}
1264	#endif /* VBOX_STRICT */
1265
1266
1267	/**
1268	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1269	* area.
1270	*
1271	* @returns VBox status code.
1272	* @param pVCpu The cross context virtual CPU structure.
1273	* @param cMsrs The number of MSRs.
1274	*/
1275	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1276	{
1277	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1278	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1279	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1280	{
1281	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1282	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1283	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1284	}
1285
1286	/* Update number of guest MSRs to load/store across the world-switch. */
1287	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1288	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1289
1290	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1291	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1292	AssertRCReturn(rc, rc);
1293
1294	/* Update the VCPU's copy of the MSR count. */
1295	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1296
1297	return VINF_SUCCESS;
1298	}
1299
1300
1301	/**
1302	* Adds a new (or updates the value of an existing) guest/host MSR
1303	* pair to be swapped during the world-switch as part of the
1304	* auto-load/store MSR area in the VMCS.
1305	*
1306	* @returns VBox status code.
1307	* @param pVCpu The cross context virtual CPU structure.
1308	* @param uMsr The MSR.
1309	* @param uGuestMsrValue Value of the guest MSR.
1310	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1311	* necessary.
1312	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1313	* its value was updated. Optional, can be NULL.
1314	*/
1315	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1316	bool *pfAddedAndUpdated)
1317	{
1318	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1319	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1320	uint32_t i;
1321	for (i = 0; i < cMsrs; i++)
1322	{
1323	if (pGuestMsr->u32Msr == uMsr)
1324	break;
1325	pGuestMsr++;
1326	}
1327
1328	bool fAdded = false;
1329	if (i == cMsrs)
1330	{
1331	++cMsrs;
1332	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1333	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1334
1335	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1336	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1337	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1338
1339	fAdded = true;
1340	}
1341
1342	/* Update the MSR values in the auto-load/store MSR area. */
1343	pGuestMsr->u32Msr = uMsr;
1344	pGuestMsr->u64Value = uGuestMsrValue;
1345
1346	/* Create/update the MSR slot in the host MSR area. */
1347	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348	pHostMsr += i;
1349	pHostMsr->u32Msr = uMsr;
1350
1351	/*
1352	* Update the host MSR only when requested by the caller AND when we're
1353	* adding it to the auto-load/store area. Otherwise, it would have been
1354	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1355	*/
1356	bool fUpdatedMsrValue = false;
1357	if ( fAdded
1358	&& fUpdateHostMsr)
1359	{
1360	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1361	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1362	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1363	fUpdatedMsrValue = true;
1364	}
1365
1366	if (pfAddedAndUpdated)
1367	*pfAddedAndUpdated = fUpdatedMsrValue;
1368	return VINF_SUCCESS;
1369	}
1370
1371
1372	/**
1373	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1374	* auto-load/store MSR area in the VMCS.
1375	*
1376	* @returns VBox status code.
1377	* @param pVCpu The cross context virtual CPU structure.
1378	* @param uMsr The MSR.
1379	*/
1380	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1381	{
1382	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1383	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1384	for (uint32_t i = 0; i < cMsrs; i++)
1385	{
1386	/* Find the MSR. */
1387	if (pGuestMsr->u32Msr == uMsr)
1388	{
1389	/* If it's the last MSR, simply reduce the count. */
1390	if (i == cMsrs - 1)
1391	{
1392	--cMsrs;
1393	break;
1394	}
1395
1396	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1397	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1398	pLastGuestMsr += cMsrs - 1;
1399	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1400	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1401
1402	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1403	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404	pLastHostMsr += cMsrs - 1;
1405	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1406	pHostMsr->u64Value = pLastHostMsr->u64Value;
1407	--cMsrs;
1408	break;
1409	}
1410	pGuestMsr++;
1411	}
1412
1413	/* Update the VMCS if the count changed (meaning the MSR was found). */
1414	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1415	{
1416	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1417	AssertRCReturn(rc, rc);
1418
1419	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1420	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1421	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1422
1423	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1424	return VINF_SUCCESS;
1425	}
1426
1427	return VERR_NOT_FOUND;
1428	}
1429
1430
1431	/**
1432	* Checks if the specified guest MSR is part of the auto-load/store area in
1433	* the VMCS.
1434	*
1435	* @returns true if found, false otherwise.
1436	* @param pVCpu The cross context virtual CPU structure.
1437	* @param uMsr The MSR to find.
1438	*/
1439	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1440	{
1441	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1445	{
1446	if (pGuestMsr->u32Msr == uMsr)
1447	return true;
1448	}
1449	return false;
1450	}
1451
1452
1453	/**
1454	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1455	*
1456	* @param pVCpu The cross context virtual CPU structure.
1457	*
1458	* @remarks No-long-jump zone!!!
1459	*/
1460	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1461	{
1462	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1463	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1464	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1465	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1466
1467	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1468	{
1469	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1470
1471	/*
1472	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1473	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1474	*/
1475	if (pHostMsr->u32Msr == MSR_K6_EFER)
1476	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1477	else
1478	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1479	}
1480
1481	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1482	}
1483
1484
1485	/**
1486	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1487	* perform lazy restoration of the host MSRs while leaving VT-x.
1488	*
1489	* @param pVCpu The cross context virtual CPU structure.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496
1497	/*
1498	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1499	*/
1500	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1501	{
1502	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1503	#if HC_ARCH_BITS == 64
1504	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505	{
1506	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1507	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1508	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1509	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1510	}
1511	#endif
1512	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1513	}
1514	}
1515
1516
1517	/**
1518	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1519	* lazily while leaving VT-x.
1520	*
1521	* @returns true if it does, false otherwise.
1522	* @param pVCpu The cross context virtual CPU structure.
1523	* @param uMsr The MSR to check.
1524	*/
1525	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1526	{
1527	NOREF(pVCpu);
1528	#if HC_ARCH_BITS == 64
1529	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1530	{
1531	switch (uMsr)
1532	{
1533	case MSR_K8_LSTAR:
1534	case MSR_K6_STAR:
1535	case MSR_K8_SF_MASK:
1536	case MSR_K8_KERNEL_GS_BASE:
1537	return true;
1538	}
1539	}
1540	#else
1541	RT_NOREF(pVCpu, uMsr);
1542	#endif
1543	return false;
1544	}
1545
1546
1547	/**
1548	* Saves a set of guest MSRs back into the guest-CPU context.
1549	*
1550	* @param pVCpu The cross context virtual CPU structure.
1551	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1552	* out-of-sync. Make sure to update the required fields
1553	* before using them.
1554	*
1555	* @remarks No-long-jump zone!!!
1556	*/
1557	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1558	{
1559	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1560	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1561
1562	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1563	{
1564	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1565	#if HC_ARCH_BITS == 64
1566	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1567	{
1568	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1569	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1570	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1571	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1572	}
1573	#else
1574	NOREF(pMixedCtx);
1575	#endif
1576	}
1577	}
1578
1579
1580	/**
1581	* Loads a set of guests MSRs to allow read/passthru to the guest.
1582	*
1583	* The name of this function is slightly confusing. This function does NOT
1584	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1585	* common prefix for functions dealing with "lazy restoration" of the shared
1586	* MSRs.
1587	*
1588	* @param pVCpu The cross context virtual CPU structure.
1589	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1590	* out-of-sync. Make sure to update the required fields
1591	* before using them.
1592	*
1593	* @remarks No-long-jump zone!!!
1594	*/
1595	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1596	{
1597	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1598	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1599
1600	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1601	#if HC_ARCH_BITS == 64
1602	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1603	{
1604	/*
1605	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1606	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1607	* we can skip a few MSR writes.
1608	*
1609	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1610	* guest MSR values in the guest-CPU context might be different to what's currently
1611	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1612	* CPU, see @bugref{8728}.
1613	*/
1614	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1615	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1616	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1617	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1618	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1619	{
1620	#ifdef VBOX_STRICT
1621	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1622	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1623	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1624	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1625	#endif
1626	}
1627	else
1628	{
1629	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1630	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1631	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1632	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1633	}
1634	}
1635	#else
1636	RT_NOREF(pMixedCtx);
1637	#endif
1638	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1639	}
1640
1641
1642	/**
1643	* Performs lazy restoration of the set of host MSRs if they were previously
1644	* loaded with guest MSR values.
1645	*
1646	* @param pVCpu The cross context virtual CPU structure.
1647	*
1648	* @remarks No-long-jump zone!!!
1649	* @remarks The guest MSRs should have been saved back into the guest-CPU
1650	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1651	*/
1652	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1653	{
1654	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1655	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1656
1657	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1658	{
1659	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1660	#if HC_ARCH_BITS == 64
1661	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1662	{
1663	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1664	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1665	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1666	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1667	}
1668	#endif
1669	}
1670	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1671	}
1672
1673
1674	/**
1675	* Verifies that our cached values of the VMCS controls are all
1676	* consistent with what's actually present in the VMCS.
1677	*
1678	* @returns VBox status code.
1679	* @param pVCpu The cross context virtual CPU structure.
1680	*/
1681	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1682	{
1683	uint32_t u32Val;
1684	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1685	AssertRCReturn(rc, rc);
1686	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1687	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1688
1689	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1690	AssertRCReturn(rc, rc);
1691	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1692	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1693
1694	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1695	AssertRCReturn(rc, rc);
1696	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1697	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1698
1699	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1700	AssertRCReturn(rc, rc);
1701	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1702	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1703
1704	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1705	{
1706	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1707	AssertRCReturn(rc, rc);
1708	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1709	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1710	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1711	}
1712
1713	return VINF_SUCCESS;
1714	}
1715
1716
1717	#ifdef VBOX_STRICT
1718	/**
1719	* Verifies that our cached host EFER value has not changed
1720	* since we cached it.
1721	*
1722	* @param pVCpu The cross context virtual CPU structure.
1723	*/
1724	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1725	{
1726	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1727
1728	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1729	{
1730	uint64_t u64Val;
1731	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1732	AssertRC(rc);
1733
1734	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1735	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1736	}
1737	}
1738
1739
1740	/**
1741	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1742	* VMCS are correct.
1743	*
1744	* @param pVCpu The cross context virtual CPU structure.
1745	*/
1746	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1747	{
1748	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1749
1750	/* Verify MSR counts in the VMCS are what we think it should be. */
1751	uint32_t cMsrs;
1752	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1753	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1754
1755	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1756	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1757
1758	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1759	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1760
1761	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1762	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1763	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1764	{
1765	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1766	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1767	pGuestMsr->u32Msr, cMsrs));
1768
1769	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1770	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1771	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1772
1773	/* Verify that the permissions are as expected in the MSR bitmap. */
1774	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1775	{
1776	VMXMSREXITREAD enmRead;
1777	VMXMSREXITWRITE enmWrite;
1778	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1779	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1780	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1781	{
1782	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1783	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1784	}
1785	else
1786	{
1787	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1788	pGuestMsr->u32Msr, cMsrs));
1789	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1790	pGuestMsr->u32Msr, cMsrs));
1791	}
1792	}
1793	}
1794	}
1795	#endif /* VBOX_STRICT */
1796
1797
1798	/**
1799	* Flushes the TLB using EPT.
1800	*
1801	* @returns VBox status code.
1802	* @param pVCpu The cross context virtual CPU structure of the calling
1803	* EMT. Can be NULL depending on @a enmFlush.
1804	* @param enmFlush Type of flush.
1805	*
1806	* @remarks Caller is responsible for making sure this function is called only
1807	* when NestedPaging is supported and providing @a enmFlush that is
1808	* supported by the CPU.
1809	* @remarks Can be called with interrupts disabled.
1810	*/
1811	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1812	{
1813	uint64_t au64Descriptor[2];
1814	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1815	au64Descriptor[0] = 0;
1816	else
1817	{
1818	Assert(pVCpu);
1819	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1820	}
1821	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1822
1823	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1824	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1825	rc));
1826	if ( RT_SUCCESS(rc)
1827	&& pVCpu)
1828	{
1829	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1830	}
1831	}
1832
1833
1834	/**
1835	* Flushes the TLB using VPID.
1836	*
1837	* @returns VBox status code.
1838	* @param pVM The cross context VM structure.
1839	* @param pVCpu The cross context virtual CPU structure of the calling
1840	* EMT. Can be NULL depending on @a enmFlush.
1841	* @param enmFlush Type of flush.
1842	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1843	* on @a enmFlush).
1844	*
1845	* @remarks Can be called with interrupts disabled.
1846	*/
1847	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1848	{
1849	NOREF(pVM);
1850	AssertPtr(pVM);
1851	Assert(pVM->hm.s.vmx.fVpid);
1852
1853	uint64_t au64Descriptor[2];
1854	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1855	{
1856	au64Descriptor[0] = 0;
1857	au64Descriptor[1] = 0;
1858	}
1859	else
1860	{
1861	AssertPtr(pVCpu);
1862	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1863	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1864	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1865	au64Descriptor[1] = GCPtr;
1866	}
1867
1868	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1869	AssertMsg(rc == VINF_SUCCESS,
1870	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1871	if ( RT_SUCCESS(rc)
1872	&& pVCpu)
1873	{
1874	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1875	}
1876	}
1877
1878
1879	/**
1880	* Invalidates a guest page by guest virtual address. Only relevant for
1881	* EPT/VPID, otherwise there is nothing really to invalidate.
1882	*
1883	* @returns VBox status code.
1884	* @param pVM The cross context VM structure.
1885	* @param pVCpu The cross context virtual CPU structure.
1886	* @param GCVirt Guest virtual address of the page to invalidate.
1887	*/
1888	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1889	{
1890	AssertPtr(pVM);
1891	AssertPtr(pVCpu);
1892	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1893
1894	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1895	if (!fFlushPending)
1896	{
1897	/*
1898	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1899	* See @bugref{6043} and @bugref{6177}.
1900	*
1901	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1902	* function maybe called in a loop with individual addresses.
1903	*/
1904	if (pVM->hm.s.vmx.fVpid)
1905	{
1906	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1907	{
1908	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1909	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1910	}
1911	else
1912	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1913	}
1914	else if (pVM->hm.s.fNestedPaging)
1915	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1916	}
1917
1918	return VINF_SUCCESS;
1919	}
1920
1921
1922	/**
1923	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1924	* otherwise there is nothing really to invalidate.
1925	*
1926	* @returns VBox status code.
1927	* @param pVM The cross context VM structure.
1928	* @param pVCpu The cross context virtual CPU structure.
1929	* @param GCPhys Guest physical address of the page to invalidate.
1930	*/
1931	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1932	{
1933	NOREF(pVM); NOREF(GCPhys);
1934	LogFlowFunc(("%RGp\n", GCPhys));
1935
1936	/*
1937	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1938	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1939	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1940	*/
1941	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1942	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1943	return VINF_SUCCESS;
1944	}
1945
1946
1947	/**
1948	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1949	* case where neither EPT nor VPID is supported by the CPU.
1950	*
1951	* @param pVM The cross context VM structure.
1952	* @param pVCpu The cross context virtual CPU structure.
1953	* @param pCpu Pointer to the global HM struct.
1954	*
1955	* @remarks Called with interrupts disabled.
1956	*/
1957	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1958	{
1959	AssertPtr(pVCpu);
1960	AssertPtr(pCpu);
1961	NOREF(pVM);
1962
1963	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1964
1965	Assert(pCpu->idCpu != NIL_RTCPUID);
1966	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1967	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1968	pVCpu->hm.s.fForceTLBFlush = false;
1969	return;
1970	}
1971
1972
1973	/**
1974	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1975	*
1976	* @param pVM The cross context VM structure.
1977	* @param pVCpu The cross context virtual CPU structure.
1978	* @param pCpu Pointer to the global HM CPU struct.
1979	* @remarks All references to "ASID" in this function pertains to "VPID" in
1980	* Intel's nomenclature. The reason is, to avoid confusion in compare
1981	* statements since the host-CPU copies are named "ASID".
1982	*
1983	* @remarks Called with interrupts disabled.
1984	*/
1985	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1986	{
1987	#ifdef VBOX_WITH_STATISTICS
1988	bool fTlbFlushed = false;
1989	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1990	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1991	if (!fTlbFlushed) \
1992	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1993	} while (0)
1994	#else
1995	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1996	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1997	#endif
1998
1999	AssertPtr(pVM);
2000	AssertPtr(pCpu);
2001	AssertPtr(pVCpu);
2002	Assert(pCpu->idCpu != NIL_RTCPUID);
2003
2004	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2005	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2006	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2007
2008	/*
2009	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2010	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2011	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2012	*/
2013	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2014	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2015	{
2016	++pCpu->uCurrentAsid;
2017	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2018	{
2019	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2020	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2021	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2022	}
2023
2024	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2025	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2026	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2027
2028	/*
2029	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2030	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2031	*/
2032	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2033	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034	HMVMX_SET_TAGGED_TLB_FLUSHED();
2035	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2036	}
2037
2038	/* Check for explicit TLB flushes. */
2039	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2040	{
2041	/*
2042	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2043	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2044	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2045	* but not guest-physical mappings.
2046	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2047	*/
2048	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2049	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050	HMVMX_SET_TAGGED_TLB_FLUSHED();
2051	}
2052
2053	pVCpu->hm.s.fForceTLBFlush = false;
2054	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2055
2056	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2057	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2058	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2059	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2060	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2061	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2062	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2063	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2064	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2065
2066	/* Update VMCS with the VPID. */
2067	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2068	AssertRC(rc);
2069
2070	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2071	}
2072
2073
2074	/**
2075	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2076	*
2077	* @returns VBox status code.
2078	* @param pVM The cross context VM structure.
2079	* @param pVCpu The cross context virtual CPU structure.
2080	* @param pCpu Pointer to the global HM CPU struct.
2081	*
2082	* @remarks Called with interrupts disabled.
2083	*/
2084	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	AssertPtr(pVM);
2087	AssertPtr(pVCpu);
2088	AssertPtr(pCpu);
2089	Assert(pCpu->idCpu != NIL_RTCPUID);
2090	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2091	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2092
2093	/*
2094	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2095	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2096	*/
2097	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2098	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2099	{
2100	pVCpu->hm.s.fForceTLBFlush = true;
2101	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2102	}
2103
2104	/* Check for explicit TLB flushes. */
2105	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2106	{
2107	pVCpu->hm.s.fForceTLBFlush = true;
2108	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2109	}
2110
2111	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2112	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2113
2114	if (pVCpu->hm.s.fForceTLBFlush)
2115	{
2116	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2117	pVCpu->hm.s.fForceTLBFlush = false;
2118	}
2119	}
2120
2121
2122	/**
2123	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2124	*
2125	* @returns VBox status code.
2126	* @param pVM The cross context VM structure.
2127	* @param pVCpu The cross context virtual CPU structure.
2128	* @param pCpu Pointer to the global HM CPU struct.
2129	*
2130	* @remarks Called with interrupts disabled.
2131	*/
2132	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2133	{
2134	AssertPtr(pVM);
2135	AssertPtr(pVCpu);
2136	AssertPtr(pCpu);
2137	Assert(pCpu->idCpu != NIL_RTCPUID);
2138	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2139	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2140
2141	/*
2142	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2143	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2144	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2145	*/
2146	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2147	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2148	{
2149	pVCpu->hm.s.fForceTLBFlush = true;
2150	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2151	}
2152
2153	/* Check for explicit TLB flushes. */
2154	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2155	{
2156	/*
2157	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2158	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2159	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2160	*/
2161	pVCpu->hm.s.fForceTLBFlush = true;
2162	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2163	}
2164
2165	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2166	if (pVCpu->hm.s.fForceTLBFlush)
2167	{
2168	++pCpu->uCurrentAsid;
2169	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2170	{
2171	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2172	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2173	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2174	}
2175
2176	pVCpu->hm.s.fForceTLBFlush = false;
2177	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2178	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2179	if (pCpu->fFlushAsidBeforeUse)
2180	{
2181	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2182	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2183	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2184	{
2185	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2186	pCpu->fFlushAsidBeforeUse = false;
2187	}
2188	else
2189	{
2190	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2191	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2192	}
2193	}
2194	}
2195
2196	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2197	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2198	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2199	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2200	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2201	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2202	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2203
2204	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2205	AssertRC(rc);
2206	}
2207
2208
2209	/**
2210	* Flushes the guest TLB entry based on CPU capabilities.
2211	*
2212	* @param pVCpu The cross context virtual CPU structure.
2213	* @param pCpu Pointer to the global HM CPU struct.
2214	*/
2215	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2216	{
2217	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2218	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2219	#endif
2220	PVM pVM = pVCpu->CTX_SUFF(pVM);
2221	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2222	{
2223	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2224	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2225	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2226	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2227	default:
2228	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2229	break;
2230	}
2231
2232	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2233	}
2234
2235
2236	/**
2237	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2238	* TLB entries from the host TLB before VM-entry.
2239	*
2240	* @returns VBox status code.
2241	* @param pVM The cross context VM structure.
2242	*/
2243	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2244	{
2245	/*
2246	* Determine optimal flush type for Nested Paging.
2247	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2248	* guest execution (see hmR3InitFinalizeR0()).
2249	*/
2250	if (pVM->hm.s.fNestedPaging)
2251	{
2252	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2253	{
2254	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2255	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2256	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2257	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2258	else
2259	{
2260	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2261	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2263	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2264	}
2265
2266	/* Make sure the write-back cacheable memory type for EPT is supported. */
2267	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2268	{
2269	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2270	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2271	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2272	}
2273
2274	/* EPT requires a page-walk length of 4. */
2275	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2276	{
2277	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2278	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2279	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2280	}
2281	}
2282	else
2283	{
2284	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2285	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2286	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2287	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2288	}
2289	}
2290
2291	/*
2292	* Determine optimal flush type for VPID.
2293	*/
2294	if (pVM->hm.s.vmx.fVpid)
2295	{
2296	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2297	{
2298	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2299	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2300	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2301	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2302	else
2303	{
2304	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2305	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2306	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2307	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2308	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2309	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2310	pVM->hm.s.vmx.fVpid = false;
2311	}
2312	}
2313	else
2314	{
2315	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2316	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2317	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2318	pVM->hm.s.vmx.fVpid = false;
2319	}
2320	}
2321
2322	/*
2323	* Setup the handler for flushing tagged-TLBs.
2324	*/
2325	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2326	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2327	else if (pVM->hm.s.fNestedPaging)
2328	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2329	else if (pVM->hm.s.vmx.fVpid)
2330	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2331	else
2332	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2333	return VINF_SUCCESS;
2334	}
2335
2336
2337	/**
2338	* Sets up pin-based VM-execution controls in the VMCS.
2339	*
2340	* @returns VBox status code.
2341	* @param pVM The cross context VM structure.
2342	* @param pVCpu The cross context virtual CPU structure.
2343	*/
2344	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2345	{
2346	AssertPtr(pVM);
2347	AssertPtr(pVCpu);
2348
2349	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2350	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2351
2352	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2353	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2354
2355	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2356	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2357
2358	/* Enable the VMX preemption timer. */
2359	if (pVM->hm.s.vmx.fUsePreemptTimer)
2360	{
2361	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2362	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2363	}
2364
2365	#if 0
2366	/* Enable posted-interrupt processing. */
2367	if (pVM->hm.s.fPostedIntrs)
2368	{
2369	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2370	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2371	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2372	}
2373	#endif
2374
2375	if ((val & zap) != val)
2376	{
2377	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2378	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2379	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2380	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2381	}
2382
2383	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2384	AssertRCReturn(rc, rc);
2385
2386	pVCpu->hm.s.vmx.u32PinCtls = val;
2387	return rc;
2388	}
2389
2390
2391	/**
2392	* Sets up processor-based VM-execution controls in the VMCS.
2393	*
2394	* @returns VBox status code.
2395	* @param pVM The cross context VM structure.
2396	* @param pVCpu The cross context virtual CPU structure.
2397	*/
2398	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2399	{
2400	AssertPtr(pVM);
2401	AssertPtr(pVCpu);
2402
2403	int rc = VERR_INTERNAL_ERROR_5;
2404	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2405	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2406
2407	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2408	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2409	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2410	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2411	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2412	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2413	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2414
2415	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2416	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2417	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2418	{
2419	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2420	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2421	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2422	}
2423
2424	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2425	if (!pVM->hm.s.fNestedPaging)
2426	{
2427	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2428	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2429	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2430	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2431	}
2432
2433	/* Use TPR shadowing if supported by the CPU. */
2434	if ( PDMHasApic(pVM)
2435	&& pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2436	{
2437	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2438	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2439	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2440	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2441	AssertRCReturn(rc, rc);
2442
2443	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2444	/* CR8 writes cause a VM-exit based on TPR threshold. */
2445	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2446	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2447	}
2448	else
2449	{
2450	/*
2451	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2452	* Set this control only for 64-bit guests.
2453	*/
2454	if (pVM->hm.s.fAllow64BitGuests)
2455	{
2456	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2457	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2458	}
2459	}
2460
2461	/* Use MSR-bitmaps if supported by the CPU. */
2462	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2463	{
2464	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2465
2466	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2467	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2468	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2469	AssertRCReturn(rc, rc);
2470
2471	/*
2472	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2473	* automatically using dedicated fields in the VMCS.
2474	*/
2475	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2476	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2477	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2478	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2479	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2480
2481	#if HC_ARCH_BITS == 64
2482	/*
2483	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2484	*/
2485	if (pVM->hm.s.fAllow64BitGuests)
2486	{
2487	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2488	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2489	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2490	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2491	}
2492	#endif
2493	/*
2494	* The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2495	* access (writes need to be executed without exiting, reds will #GP-fault anyway).
2496	*/
2497	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2498	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2499
2500	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2501	}
2502
2503	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2504	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2505	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2506
2507	if ((val & zap) != val)
2508	{
2509	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2510	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2511	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2512	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2513	}
2514
2515	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2516	AssertRCReturn(rc, rc);
2517
2518	pVCpu->hm.s.vmx.u32ProcCtls = val;
2519
2520	/*
2521	* Secondary processor-based VM-execution controls.
2522	*/
2523	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2524	{
2525	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2526	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2527
2528	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2529	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2530
2531	if (pVM->hm.s.fNestedPaging)
2532	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2533
2534	/*
2535	* Enable the INVPCID instruction if supported by the hardware and we expose
2536	* it to the guest. Without this, guest executing INVPCID would cause a #UD.
2537	*/
2538	if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2539	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2540	{
2541	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2542	}
2543
2544	if (pVM->hm.s.vmx.fVpid)
2545	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2546
2547	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2548	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2549
2550	#if 0
2551	if (pVM->hm.s.fVirtApicRegs)
2552	{
2553	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2554	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2555
2556	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2557	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2558	}
2559	#endif
2560
2561	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2562	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2563	* done dynamically. */
2564	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2565	{
2566	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2567	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2568	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2569	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2570	AssertRCReturn(rc, rc);
2571	}
2572
2573	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2574	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2575
2576	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2577	&& pVM->hm.s.vmx.cPleGapTicks
2578	&& pVM->hm.s.vmx.cPleWindowTicks)
2579	{
2580	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2581
2582	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2583	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2584	AssertRCReturn(rc, rc);
2585	}
2586
2587	if ((val & zap) != val)
2588	{
2589	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2590	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2591	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2592	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2593	}
2594
2595	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2596	AssertRCReturn(rc, rc);
2597
2598	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2599	}
2600	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2601	{
2602	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2603	"available\n"));
2604	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2605	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2606	}
2607
2608	return VINF_SUCCESS;
2609	}
2610
2611
2612	/**
2613	* Sets up miscellaneous (everything other than Pin & Processor-based
2614	* VM-execution) control fields in the VMCS.
2615	*
2616	* @returns VBox status code.
2617	* @param pVM The cross context VM structure.
2618	* @param pVCpu The cross context virtual CPU structure.
2619	*/
2620	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2621	{
2622	NOREF(pVM);
2623	AssertPtr(pVM);
2624	AssertPtr(pVCpu);
2625
2626	int rc = VERR_GENERAL_FAILURE;
2627
2628	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2629	#if 0
2630	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2631	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2632	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2633
2634	/*
2635	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2636	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2637	* We thus use the exception bitmap to control it rather than use both.
2638	*/
2639	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2640	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2641
2642	/** @todo Explore possibility of using IO-bitmaps. */
2643	/* All IO & IOIO instructions cause VM-exits. */
2644	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2645	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2646
2647	/* Initialize the MSR-bitmap area. */
2648	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2649	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2650	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2651	AssertRCReturn(rc, rc);
2652	#endif
2653
2654	/* Setup MSR auto-load/store area. */
2655	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2656	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2657	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2658	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2659	AssertRCReturn(rc, rc);
2660
2661	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2662	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2663	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2664	AssertRCReturn(rc, rc);
2665
2666	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2667	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2668	AssertRCReturn(rc, rc);
2669
2670	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2671	#if 0
2672	/* Setup debug controls */
2673	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2674	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2675	AssertRCReturn(rc, rc);
2676	#endif
2677
2678	return rc;
2679	}
2680
2681
2682	/**
2683	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2684	*
2685	* We shall setup those exception intercepts that don't change during the
2686	* lifetime of the VM here. The rest are done dynamically while loading the
2687	* guest state.
2688	*
2689	* @returns VBox status code.
2690	* @param pVM The cross context VM structure.
2691	* @param pVCpu The cross context virtual CPU structure.
2692	*/
2693	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2694	{
2695	AssertPtr(pVM);
2696	AssertPtr(pVCpu);
2697
2698	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2699
2700	uint32_t u32XcptBitmap = 0;
2701
2702	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2703	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2704
2705	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2706	and writes, and because recursive #DBs can cause the CPU hang, we must always
2707	intercept #DB. */
2708	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2709
2710	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2711	if (!pVM->hm.s.fNestedPaging)
2712	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2713
2714	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2715	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2716	AssertRCReturn(rc, rc);
2717	return rc;
2718	}
2719
2720
2721	/**
2722	* Sets up the initial guest-state mask. The guest-state mask is consulted
2723	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2724	* for the nested virtualization case (as it would cause a VM-exit).
2725	*
2726	* @param pVCpu The cross context virtual CPU structure.
2727	*/
2728	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2729	{
2730	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2731	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2732	return VINF_SUCCESS;
2733	}
2734
2735
2736	/**
2737	* Does per-VM VT-x initialization.
2738	*
2739	* @returns VBox status code.
2740	* @param pVM The cross context VM structure.
2741	*/
2742	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2743	{
2744	LogFlowFunc(("pVM=%p\n", pVM));
2745
2746	int rc = hmR0VmxStructsAlloc(pVM);
2747	if (RT_FAILURE(rc))
2748	{
2749	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2750	return rc;
2751	}
2752
2753	return VINF_SUCCESS;
2754	}
2755
2756
2757	/**
2758	* Does per-VM VT-x termination.
2759	*
2760	* @returns VBox status code.
2761	* @param pVM The cross context VM structure.
2762	*/
2763	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2764	{
2765	LogFlowFunc(("pVM=%p\n", pVM));
2766
2767	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2768	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2769	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2770	#endif
2771	hmR0VmxStructsFree(pVM);
2772	return VINF_SUCCESS;
2773	}
2774
2775
2776	/**
2777	* Sets up the VM for execution under VT-x.
2778	* This function is only called once per-VM during initialization.
2779	*
2780	* @returns VBox status code.
2781	* @param pVM The cross context VM structure.
2782	*/
2783	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2784	{
2785	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2786	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2787
2788	LogFlowFunc(("pVM=%p\n", pVM));
2789
2790	/*
2791	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2792	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2793	*/
2794	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2795	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2796	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2797	{
2798	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2799	return VERR_INTERNAL_ERROR;
2800	}
2801
2802	/* Initialize these always, see hmR3InitFinalizeR0().*/
2803	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2804	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2805
2806	/* Setup the tagged-TLB flush handlers. */
2807	int rc = hmR0VmxSetupTaggedTlb(pVM);
2808	if (RT_FAILURE(rc))
2809	{
2810	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2811	return rc;
2812	}
2813
2814	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2815	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2816	#if HC_ARCH_BITS == 64
2817	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2818	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2819	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2820	{
2821	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2822	}
2823	#endif
2824
2825	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2826	RTCCUINTREG uHostCR4 = ASMGetCR4();
2827	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2828	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2829
2830	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2831	{
2832	PVMCPU pVCpu = &pVM->aCpus[i];
2833	AssertPtr(pVCpu);
2834	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2835
2836	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2837	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2838
2839	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2840	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2841	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2842
2843	/* Set revision dword at the beginning of the VMCS structure. */
2844	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2845
2846	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2847	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2848	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2849	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2850
2851	/* Load this VMCS as the current VMCS. */
2852	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2853	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2854	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2855
2856	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2857	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2858	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2859
2860	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2861	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2862	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2863
2864	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2865	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2866	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2867
2868	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2869	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2870	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2871
2872	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2873	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2874	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2875
2876	#if HC_ARCH_BITS == 32
2877	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2878	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2879	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2880	#endif
2881
2882	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2883	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2884	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2885	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2886
2887	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2888
2889	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2890	}
2891
2892	return VINF_SUCCESS;
2893	}
2894
2895
2896	/**
2897	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2898	* the VMCS.
2899	*
2900	* @returns VBox status code.
2901	* @param pVM The cross context VM structure.
2902	* @param pVCpu The cross context virtual CPU structure.
2903	*/
2904	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2905	{
2906	NOREF(pVM); NOREF(pVCpu);
2907
2908	RTCCUINTREG uReg = ASMGetCR0();
2909	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2910	AssertRCReturn(rc, rc);
2911
2912	uReg = ASMGetCR3();
2913	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2914	AssertRCReturn(rc, rc);
2915
2916	uReg = ASMGetCR4();
2917	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2918	AssertRCReturn(rc, rc);
2919	return rc;
2920	}
2921
2922
2923	#if HC_ARCH_BITS == 64
2924	/**
2925	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2926	* requirements. See hmR0VmxSaveHostSegmentRegs().
2927	*/
2928	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2929	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2930	{ \
2931	bool fValidSelector = true; \
2932	if ((selValue) & X86_SEL_LDT) \
2933	{ \
2934	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2935	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2936	} \
2937	if (fValidSelector) \
2938	{ \
2939	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2940	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2941	} \
2942	(selValue) = 0; \
2943	}
2944	#endif
2945
2946
2947	/**
2948	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2949	* the host-state area in the VMCS.
2950	*
2951	* @returns VBox status code.
2952	* @param pVM The cross context VM structure.
2953	* @param pVCpu The cross context virtual CPU structure.
2954	*/
2955	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2956	{
2957	int rc = VERR_INTERNAL_ERROR_5;
2958
2959	#if HC_ARCH_BITS == 64
2960	/*
2961	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2962	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2963	*
2964	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2965	* Was observed booting Solaris10u10 32-bit guest.
2966	*/
2967	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2968	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2969	{
2970	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2971	pVCpu->idCpu));
2972	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2973	}
2974	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2975	#else
2976	RT_NOREF(pVCpu);
2977	#endif
2978
2979	/*
2980	* Host DS, ES, FS and GS segment registers.
2981	*/
2982	#if HC_ARCH_BITS == 64
2983	RTSEL uSelDS = ASMGetDS();
2984	RTSEL uSelES = ASMGetES();
2985	RTSEL uSelFS = ASMGetFS();
2986	RTSEL uSelGS = ASMGetGS();
2987	#else
2988	RTSEL uSelDS = 0;
2989	RTSEL uSelES = 0;
2990	RTSEL uSelFS = 0;
2991	RTSEL uSelGS = 0;
2992	#endif
2993
2994	/*
2995	* Host CS and SS segment registers.
2996	*/
2997	RTSEL uSelCS = ASMGetCS();
2998	RTSEL uSelSS = ASMGetSS();
2999
3000	/*
3001	* Host TR segment register.
3002	*/
3003	RTSEL uSelTR = ASMGetTR();
3004
3005	#if HC_ARCH_BITS == 64
3006	/*
3007	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3008	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3009	*/
3010	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3011	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3012	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3013	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3014	# undef VMXLOCAL_ADJUST_HOST_SEG
3015	#endif
3016
3017	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3018	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3019	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3020	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3021	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3022	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3023	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3024	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3025	Assert(uSelCS);
3026	Assert(uSelTR);
3027
3028	/* Assertion is right but we would not have updated u32ExitCtls yet. */
3029	#if 0
3030	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3031	Assert(uSelSS != 0);
3032	#endif
3033
3034	/* Write these host selector fields into the host-state area in the VMCS. */
3035	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3036	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3037	#if HC_ARCH_BITS == 64
3038	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3039	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3040	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3041	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3042	#else
3043	NOREF(uSelDS);
3044	NOREF(uSelES);
3045	NOREF(uSelFS);
3046	NOREF(uSelGS);
3047	#endif
3048	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3049	AssertRCReturn(rc, rc);
3050
3051	/*
3052	* Host GDTR and IDTR.
3053	*/
3054	RTGDTR Gdtr;
3055	RTIDTR Idtr;
3056	RT_ZERO(Gdtr);
3057	RT_ZERO(Idtr);
3058	ASMGetGDTR(&Gdtr);
3059	ASMGetIDTR(&Idtr);
3060	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3061	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3062	AssertRCReturn(rc, rc);
3063
3064	#if HC_ARCH_BITS == 64
3065	/*
3066	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3067	* maximum limit (0xffff) on every VM-exit.
3068	*/
3069	if (Gdtr.cbGdt != 0xffff)
3070	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3071
3072	/*
3073	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3074	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3075	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3076	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3077	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3078	* hosts where we are pretty sure it won't cause trouble.
3079	*/
3080	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3081	if (Idtr.cbIdt < 0x0fff)
3082	# else
3083	if (Idtr.cbIdt != 0xffff)
3084	# endif
3085	{
3086	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3087	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3088	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3089	}
3090	#endif
3091
3092	/*
3093	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3094	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3095	*/
3096	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3097	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3098	VERR_VMX_INVALID_HOST_STATE);
3099
3100	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3101	#if HC_ARCH_BITS == 64
3102	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3103
3104	/*
3105	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3106	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3107	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3108	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3109	*
3110	* [1] See Intel spec. 3.5 "System Descriptor Types".
3111	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3112	*/
3113	Assert(pDesc->System.u4Type == 11);
3114	if ( pDesc->System.u16LimitLow != 0x67
3115	\|\| pDesc->System.u4LimitHigh)
3116	{
3117	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3118	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3119	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3120	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3121	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3122	}
3123
3124	/*
3125	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3126	*/
3127	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
3128	{
3129	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3130	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3131	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3132	{
3133	/* The GDT is read-only but the writable GDT is available. */
3134	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3135	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3136	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3137	AssertRCReturn(rc, rc);
3138	}
3139	}
3140	#else
3141	NOREF(pVM);
3142	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3143	#endif
3144	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3145	AssertRCReturn(rc, rc);
3146
3147	/*
3148	* Host FS base and GS base.
3149	*/
3150	#if HC_ARCH_BITS == 64
3151	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3152	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3153	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3154	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3155	AssertRCReturn(rc, rc);
3156
3157	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3158	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3159	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3160	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3161	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3162	#endif
3163	return rc;
3164	}
3165
3166
3167	/**
3168	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3169	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3170	* the host after every successful VM-exit.
3171	*
3172	* @returns VBox status code.
3173	* @param pVM The cross context VM structure.
3174	* @param pVCpu The cross context virtual CPU structure.
3175	*
3176	* @remarks No-long-jump zone!!!
3177	*/
3178	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3179	{
3180	NOREF(pVM);
3181
3182	AssertPtr(pVCpu);
3183	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3184
3185	/*
3186	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3187	* rather than swapping them on every VM-entry.
3188	*/
3189	hmR0VmxLazySaveHostMsrs(pVCpu);
3190
3191	/*
3192	* Host Sysenter MSRs.
3193	*/
3194	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3195	#if HC_ARCH_BITS == 32
3196	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3197	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3198	#else
3199	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3200	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3201	#endif
3202	AssertRCReturn(rc, rc);
3203
3204	/*
3205	* Host EFER MSR.
3206	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3207	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3208	*/
3209	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3210	{
3211	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3212	AssertRCReturn(rc, rc);
3213	}
3214
3215	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3216	* hmR0VmxLoadGuestExitCtls() !! */
3217
3218	return rc;
3219	}
3220
3221
3222	/**
3223	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3224	*
3225	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3226	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3227	* hmR0VMxLoadGuestEntryCtls().
3228	*
3229	* @returns true if we need to load guest EFER, false otherwise.
3230	* @param pVCpu The cross context virtual CPU structure.
3231	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3232	* out-of-sync. Make sure to update the required fields
3233	* before using them.
3234	*
3235	* @remarks Requires EFER, CR4.
3236	* @remarks No-long-jump zone!!!
3237	*/
3238	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3239	{
3240	#ifdef HMVMX_ALWAYS_SWAP_EFER
3241	return true;
3242	#endif
3243
3244	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3245	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3246	if (CPUMIsGuestInLongMode(pVCpu))
3247	return false;
3248	#endif
3249
3250	PVM pVM = pVCpu->CTX_SUFF(pVM);
3251	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3252	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3253
3254	/*
3255	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3256	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3257	*/
3258	if ( CPUMIsGuestInLongMode(pVCpu)
3259	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3260	{
3261	return true;
3262	}
3263
3264	/*
3265	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3266	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3267	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3268	*/
3269	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3270	&& (pMixedCtx->cr0 & X86_CR0_PG)
3271	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3272	{
3273	/* Assert that host is PAE capable. */
3274	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3275	return true;
3276	}
3277
3278	/** @todo Check the latest Intel spec. for any other bits,
3279	* like SMEP/SMAP? */
3280	return false;
3281	}
3282
3283
3284	/**
3285	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3286	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3287	* controls".
3288	*
3289	* @returns VBox status code.
3290	* @param pVCpu The cross context virtual CPU structure.
3291	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3292	* out-of-sync. Make sure to update the required fields
3293	* before using them.
3294	*
3295	* @remarks Requires EFER.
3296	* @remarks No-long-jump zone!!!
3297	*/
3298	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3299	{
3300	int rc = VINF_SUCCESS;
3301	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3302	{
3303	PVM pVM = pVCpu->CTX_SUFF(pVM);
3304	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3305	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3306
3307	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3308	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3309
3310	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3311	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3312	{
3313	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3314	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3315	}
3316	else
3317	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3318
3319	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3320	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3321	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3322	{
3323	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3324	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3325	}
3326
3327	/*
3328	* The following should -not- be set (since we're not in SMM mode):
3329	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3330	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3331	*/
3332
3333	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3334	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3335
3336	if ((val & zap) != val)
3337	{
3338	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3339	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3340	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3341	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3342	}
3343
3344	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3345	AssertRCReturn(rc, rc);
3346
3347	pVCpu->hm.s.vmx.u32EntryCtls = val;
3348	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3349	}
3350	return rc;
3351	}
3352
3353
3354	/**
3355	* Sets up the VM-exit controls in the VMCS.
3356	*
3357	* @returns VBox status code.
3358	* @param pVCpu The cross context virtual CPU structure.
3359	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3360	* out-of-sync. Make sure to update the required fields
3361	* before using them.
3362	*
3363	* @remarks Requires EFER.
3364	*/
3365	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3366	{
3367	NOREF(pMixedCtx);
3368
3369	int rc = VINF_SUCCESS;
3370	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3371	{
3372	PVM pVM = pVCpu->CTX_SUFF(pVM);
3373	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3374	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3375
3376	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3377	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3378
3379	/*
3380	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3381	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3382	*/
3383	#if HC_ARCH_BITS == 64
3384	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386	#else
3387	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3388	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3389	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3390	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3391	{
3392	/* The switcher returns to long mode, EFER is managed by the switcher. */
3393	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3394	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3395	}
3396	else
3397	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3398	#endif
3399
3400	/* If the newer VMCS fields for managing EFER exists, use it. */
3401	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3402	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3403	{
3404	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3405	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3406	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3407	}
3408
3409	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3410	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3411
3412	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3413	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3414	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3415
3416	if ( pVM->hm.s.vmx.fUsePreemptTimer
3417	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3418	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3419
3420	if ((val & zap) != val)
3421	{
3422	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3423	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3424	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3425	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3426	}
3427
3428	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3429	AssertRCReturn(rc, rc);
3430
3431	pVCpu->hm.s.vmx.u32ExitCtls = val;
3432	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3433	}
3434	return rc;
3435	}
3436
3437
3438	/**
3439	* Sets the TPR threshold in the VMCS.
3440	*
3441	* @returns VBox status code.
3442	* @param pVCpu The cross context virtual CPU structure.
3443	* @param u32TprThreshold The TPR threshold (task-priority class only).
3444	*/
3445	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3446	{
3447	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3448	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3449	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3450	}
3451
3452
3453	/**
3454	* Loads the guest APIC and related state.
3455	*
3456	* @returns VBox status code.
3457	* @param pVCpu The cross context virtual CPU structure.
3458	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3459	* out-of-sync. Make sure to update the required fields
3460	* before using them.
3461	*
3462	* @remarks No-long-jump zone!!!
3463	*/
3464	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3465	{
3466	NOREF(pMixedCtx);
3467
3468	int rc = VINF_SUCCESS;
3469	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3470	{
3471	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3472	&& APICIsEnabled(pVCpu))
3473	{
3474	/*
3475	* Setup TPR shadowing.
3476	*/
3477	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3478	{
3479	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3480
3481	bool fPendingIntr = false;
3482	uint8_t u8Tpr = 0;
3483	uint8_t u8PendingIntr = 0;
3484	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3485	AssertRCReturn(rc, rc);
3486
3487	/*
3488	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3489	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3490	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3491	*/
3492	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3493	uint32_t u32TprThreshold = 0;
3494	if (fPendingIntr)
3495	{
3496	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3497	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3498	const uint8_t u8TprPriority = u8Tpr >> 4;
3499	if (u8PendingPriority <= u8TprPriority)
3500	u32TprThreshold = u8PendingPriority;
3501	}
3502
3503	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3504	AssertRCReturn(rc, rc);
3505	}
3506	}
3507	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3508	}
3509
3510	return rc;
3511	}
3512
3513
3514	/**
3515	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3516	*
3517	* @returns Guest's interruptibility-state.
3518	* @param pVCpu The cross context virtual CPU structure.
3519	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3520	* out-of-sync. Make sure to update the required fields
3521	* before using them.
3522	*
3523	* @remarks No-long-jump zone!!!
3524	*/
3525	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3526	{
3527	/*
3528	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3529	*/
3530	uint32_t uIntrState = 0;
3531	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3532	{
3533	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3534	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3535	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3536	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3537	{
3538	if (pMixedCtx->eflags.Bits.u1IF)
3539	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3540	else
3541	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3542	}
3543	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3544	{
3545	/*
3546	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3547	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3548	*/
3549	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3550	}
3551	}
3552
3553	/*
3554	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3555	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3556	* setting this would block host-NMIs and IRET will not clear the blocking.
3557	*
3558	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3559	*/
3560	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3561	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3562	{
3563	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3564	}
3565
3566	return uIntrState;
3567	}
3568
3569
3570	/**
3571	* Loads the guest's interruptibility-state into the guest-state area in the
3572	* VMCS.
3573	*
3574	* @returns VBox status code.
3575	* @param pVCpu The cross context virtual CPU structure.
3576	* @param uIntrState The interruptibility-state to set.
3577	*/
3578	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3579	{
3580	NOREF(pVCpu);
3581	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3582	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3583	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3584	AssertRC(rc);
3585	return rc;
3586	}
3587
3588
3589	/**
3590	* Loads the exception intercepts required for guest execution in the VMCS.
3591	*
3592	* @returns VBox status code.
3593	* @param pVCpu The cross context virtual CPU structure.
3594	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3595	* out-of-sync. Make sure to update the required fields
3596	* before using them.
3597	*/
3598	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3599	{
3600	NOREF(pMixedCtx);
3601	int rc = VINF_SUCCESS;
3602	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3603	{
3604	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3605	if (pVCpu->hm.s.fGIMTrapXcptUD)
3606	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3607	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3608	else
3609	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3610	#endif
3611
3612	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3613	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3614
3615	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3616	AssertRCReturn(rc, rc);
3617
3618	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3619	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3620	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3621	}
3622	return rc;
3623	}
3624
3625
3626	/**
3627	* Loads the guest's RIP into the guest-state area in the VMCS.
3628	*
3629	* @returns VBox status code.
3630	* @param pVCpu The cross context virtual CPU structure.
3631	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3632	* out-of-sync. Make sure to update the required fields
3633	* before using them.
3634	*
3635	* @remarks No-long-jump zone!!!
3636	*/
3637	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3638	{
3639	int rc = VINF_SUCCESS;
3640	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3641	{
3642	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3643	AssertRCReturn(rc, rc);
3644
3645	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3646	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3647	HMCPU_CF_VALUE(pVCpu)));
3648	}
3649	return rc;
3650	}
3651
3652
3653	/**
3654	* Loads the guest's RSP into the guest-state area in the VMCS.
3655	*
3656	* @returns VBox status code.
3657	* @param pVCpu The cross context virtual CPU structure.
3658	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3659	* out-of-sync. Make sure to update the required fields
3660	* before using them.
3661	*
3662	* @remarks No-long-jump zone!!!
3663	*/
3664	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3665	{
3666	int rc = VINF_SUCCESS;
3667	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3668	{
3669	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3670	AssertRCReturn(rc, rc);
3671
3672	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3673	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3674	}
3675	return rc;
3676	}
3677
3678
3679	/**
3680	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3681	*
3682	* @returns VBox status code.
3683	* @param pVCpu The cross context virtual CPU structure.
3684	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3685	* out-of-sync. Make sure to update the required fields
3686	* before using them.
3687	*
3688	* @remarks No-long-jump zone!!!
3689	*/
3690	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3691	{
3692	int rc = VINF_SUCCESS;
3693	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3694	{
3695	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3696	Let us assert it as such and use 32-bit VMWRITE. */
3697	Assert(!(pMixedCtx->rflags.u64 >> 32));
3698	X86EFLAGS Eflags = pMixedCtx->eflags;
3699	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3700	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3701	* These will never be cleared/set, unless some other part of the VMM
3702	* code is buggy - in which case we're better of finding and fixing
3703	* those bugs than hiding them. */
3704	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3705	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3706	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3707	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3708
3709	/*
3710	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3711	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3712	*/
3713	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3714	{
3715	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3716	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3717	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3718	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3719	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3720	}
3721
3722	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3723	AssertRCReturn(rc, rc);
3724
3725	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3726	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3727	}
3728	return rc;
3729	}
3730
3731
3732	/**
3733	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3734	*
3735	* @returns VBox status code.
3736	* @param pVCpu The cross context virtual CPU structure.
3737	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3738	* out-of-sync. Make sure to update the required fields
3739	* before using them.
3740	*
3741	* @remarks No-long-jump zone!!!
3742	*/
3743	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3744	{
3745	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3746	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3747	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3748	AssertRCReturn(rc, rc);
3749	return rc;
3750	}
3751
3752
3753	/**
3754	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3755	* CR0 is partially shared with the host and we have to consider the FPU bits.
3756	*
3757	* @returns VBox status code.
3758	* @param pVCpu The cross context virtual CPU structure.
3759	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3760	* out-of-sync. Make sure to update the required fields
3761	* before using them.
3762	*
3763	* @remarks No-long-jump zone!!!
3764	*/
3765	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3766	{
3767	/*
3768	* Guest CR0.
3769	* Guest FPU.
3770	*/
3771	int rc = VINF_SUCCESS;
3772	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3773	{
3774	Assert(!(pMixedCtx->cr0 >> 32));
3775	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3776	PVM pVM = pVCpu->CTX_SUFF(pVM);
3777
3778	/* The guest's view (read access) of its CR0 is unblemished. */
3779	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3780	AssertRCReturn(rc, rc);
3781	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3782
3783	/* Setup VT-x's view of the guest CR0. */
3784	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3785	if (pVM->hm.s.fNestedPaging)
3786	{
3787	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3788	{
3789	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3790	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3791	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3792	}
3793	else
3794	{
3795	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3796	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3797	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3798	}
3799
3800	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3801	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3802	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3803
3804	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3805	AssertRCReturn(rc, rc);
3806	}
3807	else
3808	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3809
3810	/*
3811	* Guest FPU bits.
3812	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3813	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3814	*/
3815	u32GuestCR0 \|= X86_CR0_NE;
3816	bool fInterceptNM = false;
3817	if (CPUMIsGuestFPUStateActive(pVCpu))
3818	{
3819	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3820	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3821	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3822	}
3823	else
3824	{
3825	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3826	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3827	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3828	}
3829
3830	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3831	bool fInterceptMF = false;
3832	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3833	fInterceptMF = true;
3834
3835	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3836	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3837	{
3838	Assert(PDMVmmDevHeapIsEnabled(pVM));
3839	Assert(pVM->hm.s.vmx.pRealModeTSS);
3840	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3841	fInterceptNM = true;
3842	fInterceptMF = true;
3843	}
3844	else
3845	{
3846	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3847	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3848	}
3849	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3850
3851	if (fInterceptNM)
3852	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3853	else
3854	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3855
3856	if (fInterceptMF)
3857	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3858	else
3859	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3860
3861	/* Additional intercepts for debugging, define these yourself explicitly. */
3862	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3863	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3864	\| RT_BIT(X86_XCPT_BP)
3865	\| RT_BIT(X86_XCPT_DE)
3866	\| RT_BIT(X86_XCPT_NM)
3867	\| RT_BIT(X86_XCPT_TS)
3868	\| RT_BIT(X86_XCPT_UD)
3869	\| RT_BIT(X86_XCPT_NP)
3870	\| RT_BIT(X86_XCPT_SS)
3871	\| RT_BIT(X86_XCPT_GP)
3872	\| RT_BIT(X86_XCPT_PF)
3873	\| RT_BIT(X86_XCPT_MF)
3874	;
3875	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3876	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3877	#endif
3878
3879	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3880
3881	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3882	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3883	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3884	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3885	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3886	else
3887	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3888
3889	u32GuestCR0 \|= uSetCR0;
3890	u32GuestCR0 &= uZapCR0;
3891	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3892
3893	/* Write VT-x's view of the guest CR0 into the VMCS. */
3894	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3895	AssertRCReturn(rc, rc);
3896	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3897	uZapCR0));
3898
3899	/*
3900	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3901	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3902	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3903	*/
3904	uint32_t u32CR0Mask = 0;
3905	u32CR0Mask = X86_CR0_PE
3906	\| X86_CR0_NE
3907	\| X86_CR0_WP
3908	\| X86_CR0_PG
3909	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3910	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3911	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3912
3913	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3914	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3915	* and @bugref{6944}. */
3916	#if 0
3917	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3918	u32CR0Mask &= ~X86_CR0_PE;
3919	#endif
3920	if (pVM->hm.s.fNestedPaging)
3921	u32CR0Mask &= ~X86_CR0_WP;
3922
3923	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3924	if (fInterceptNM)
3925	{
3926	u32CR0Mask \|= X86_CR0_TS
3927	\| X86_CR0_MP;
3928	}
3929
3930	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3931	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3932	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3933	AssertRCReturn(rc, rc);
3934	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3935
3936	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3937	}
3938	return rc;
3939	}
3940
3941
3942	/**
3943	* Loads the guest control registers (CR3, CR4) into the guest-state area
3944	* in the VMCS.
3945	*
3946	* @returns VBox strict status code.
3947	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3948	* without unrestricted guest access and the VMMDev is not presently
3949	* mapped (e.g. EFI32).
3950	*
3951	* @param pVCpu The cross context virtual CPU structure.
3952	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3953	* out-of-sync. Make sure to update the required fields
3954	* before using them.
3955	*
3956	* @remarks No-long-jump zone!!!
3957	*/
3958	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3959	{
3960	int rc = VINF_SUCCESS;
3961	PVM pVM = pVCpu->CTX_SUFF(pVM);
3962
3963	/*
3964	* Guest CR2.
3965	* It's always loaded in the assembler code. Nothing to do here.
3966	*/
3967
3968	/*
3969	* Guest CR3.
3970	*/
3971	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3972	{
3973	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3974	if (pVM->hm.s.fNestedPaging)
3975	{
3976	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3977
3978	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3979	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3980	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3981	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3982
3983	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3984	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3985	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3986
3987	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3988	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3989	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3990	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3991	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3992	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3993	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3994
3995	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3996	AssertRCReturn(rc, rc);
3997	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3998
3999	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4000	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
4001	{
4002	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4003	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
4004	{
4005	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4006	AssertRCReturn(rc, rc);
4007	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4008	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4009	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4010	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4011	AssertRCReturn(rc, rc);
4012	}
4013
4014	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
4015	have Unrestricted Execution to handle the guest when it's not using paging. */
4016	GCPhysGuestCR3 = pMixedCtx->cr3;
4017	}
4018	else
4019	{
4020	/*
4021	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4022	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4023	* EPT takes care of translating it to host-physical addresses.
4024	*/
4025	RTGCPHYS GCPhys;
4026	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4027
4028	/* We obtain it here every time as the guest could have relocated this PCI region. */
4029	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4030	if (RT_SUCCESS(rc))
4031	{ /* likely */ }
4032	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4033	{
4034	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4035	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4036	}
4037	else
4038	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4039
4040	GCPhysGuestCR3 = GCPhys;
4041	}
4042
4043	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4044	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4045	}
4046	else
4047	{
4048	/* Non-nested paging case, just use the hypervisor's CR3. */
4049	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4050
4051	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4052	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4053	}
4054	AssertRCReturn(rc, rc);
4055
4056	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4057	}
4058
4059	/*
4060	* Guest CR4.
4061	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4062	*/
4063	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4064	{
4065	Assert(!(pMixedCtx->cr4 >> 32));
4066	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4067
4068	/* The guest's view of its CR4 is unblemished. */
4069	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4070	AssertRCReturn(rc, rc);
4071	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4072
4073	/* Setup VT-x's view of the guest CR4. */
4074	/*
4075	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4076	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4077	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4078	*/
4079	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4080	{
4081	Assert(pVM->hm.s.vmx.pRealModeTSS);
4082	Assert(PDMVmmDevHeapIsEnabled(pVM));
4083	u32GuestCR4 &= ~X86_CR4_VME;
4084	}
4085
4086	if (pVM->hm.s.fNestedPaging)
4087	{
4088	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4089	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4090	{
4091	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4092	u32GuestCR4 \|= X86_CR4_PSE;
4093	/* Our identity mapping is a 32-bit page directory. */
4094	u32GuestCR4 &= ~X86_CR4_PAE;
4095	}
4096	/* else use guest CR4.*/
4097	}
4098	else
4099	{
4100	/*
4101	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4102	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4103	*/
4104	switch (pVCpu->hm.s.enmShadowMode)
4105	{
4106	case PGMMODE_REAL: /* Real-mode. */
4107	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4108	case PGMMODE_32_BIT: /* 32-bit paging. */
4109	{
4110	u32GuestCR4 &= ~X86_CR4_PAE;
4111	break;
4112	}
4113
4114	case PGMMODE_PAE: /* PAE paging. */
4115	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4116	{
4117	u32GuestCR4 \|= X86_CR4_PAE;
4118	break;
4119	}
4120
4121	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4122	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4123	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4124	break;
4125	#endif
4126	default:
4127	AssertFailed();
4128	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4129	}
4130	}
4131
4132	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4133	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4134	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4135	u32GuestCR4 \|= uSetCR4;
4136	u32GuestCR4 &= uZapCR4;
4137
4138	/* Write VT-x's view of the guest CR4 into the VMCS. */
4139	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4140	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4141	AssertRCReturn(rc, rc);
4142
4143	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4144	uint32_t u32CR4Mask = X86_CR4_VME
4145	\| X86_CR4_PAE
4146	\| X86_CR4_PGE
4147	\| X86_CR4_PSE
4148	\| X86_CR4_VMXE;
4149	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4150	u32CR4Mask \|= X86_CR4_OSXSAVE;
4151	if (pVM->cpum.ro.GuestFeatures.fPcid)
4152	u32CR4Mask \|= X86_CR4_PCIDE;
4153	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4154	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4155	AssertRCReturn(rc, rc);
4156
4157	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4158	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4159
4160	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4161	}
4162	return rc;
4163	}
4164
4165
4166	/**
4167	* Loads the guest debug registers into the guest-state area in the VMCS.
4168	*
4169	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4170	*
4171	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4172	*
4173	* @returns VBox status code.
4174	* @param pVCpu The cross context virtual CPU structure.
4175	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4176	* out-of-sync. Make sure to update the required fields
4177	* before using them.
4178	*
4179	* @remarks No-long-jump zone!!!
4180	*/
4181	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4182	{
4183	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4184	return VINF_SUCCESS;
4185
4186	#ifdef VBOX_STRICT
4187	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4188	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4189	{
4190	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4191	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4192	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4193	}
4194	#endif
4195
4196	int rc;
4197	PVM pVM = pVCpu->CTX_SUFF(pVM);
4198	bool fSteppingDB = false;
4199	bool fInterceptMovDRx = false;
4200	if (pVCpu->hm.s.fSingleInstruction)
4201	{
4202	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4203	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4204	{
4205	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4206	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4207	AssertRCReturn(rc, rc);
4208	Assert(fSteppingDB == false);
4209	}
4210	else
4211	{
4212	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4213	pVCpu->hm.s.fClearTrapFlag = true;
4214	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4215	fSteppingDB = true;
4216	}
4217	}
4218
4219	if ( fSteppingDB
4220	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4221	{
4222	/*
4223	* Use the combined guest and host DRx values found in the hypervisor
4224	* register set because the debugger has breakpoints active or someone
4225	* is single stepping on the host side without a monitor trap flag.
4226	*
4227	* Note! DBGF expects a clean DR6 state before executing guest code.
4228	*/
4229	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4230	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4231	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4232	{
4233	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4234	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4235	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4236	}
4237	else
4238	#endif
4239	if (!CPUMIsHyperDebugStateActive(pVCpu))
4240	{
4241	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4242	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4243	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4244	}
4245
4246	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4247	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4248	AssertRCReturn(rc, rc);
4249
4250	pVCpu->hm.s.fUsingHyperDR7 = true;
4251	fInterceptMovDRx = true;
4252	}
4253	else
4254	{
4255	/*
4256	* If the guest has enabled debug registers, we need to load them prior to
4257	* executing guest code so they'll trigger at the right time.
4258	*/
4259	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4260	{
4261	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4262	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4263	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4264	{
4265	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4266	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4267	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4268	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4269	}
4270	else
4271	#endif
4272	if (!CPUMIsGuestDebugStateActive(pVCpu))
4273	{
4274	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4275	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4276	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4277	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4278	}
4279	Assert(!fInterceptMovDRx);
4280	}
4281	/*
4282	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4283	* must intercept #DB in order to maintain a correct DR6 guest value, and
4284	* because we need to intercept it to prevent nested #DBs from hanging the
4285	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4286	*/
4287	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4288	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4289	&& !CPUMIsGuestDebugStateActive(pVCpu))
4290	#else
4291	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4292	#endif
4293	{
4294	fInterceptMovDRx = true;
4295	}
4296
4297	/* Update guest DR7. */
4298	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4299	AssertRCReturn(rc, rc);
4300
4301	pVCpu->hm.s.fUsingHyperDR7 = false;
4302	}
4303
4304	/*
4305	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4306	*/
4307	if (fInterceptMovDRx)
4308	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4309	else
4310	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4311	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4312	AssertRCReturn(rc, rc);
4313
4314	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4315	return VINF_SUCCESS;
4316	}
4317
4318
4319	#ifdef VBOX_STRICT
4320	/**
4321	* Strict function to validate segment registers.
4322	*
4323	* @remarks ASSUMES CR0 is up to date.
4324	*/
4325	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4326	{
4327	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4328	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4329	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4330	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4331	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4332	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4333	{
4334	/* Protected mode checks */
4335	/* CS */
4336	Assert(pCtx->cs.Attr.n.u1Present);
4337	Assert(!(pCtx->cs.Attr.u & 0xf00));
4338	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4339	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4340	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4341	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4342	\|\| (pCtx->cs.Attr.n.u1Granularity));
4343	/* CS cannot be loaded with NULL in protected mode. */
4344	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4345	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4346	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4347	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4348	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4349	else
4350	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4351	/* SS */
4352	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4353	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4354	if ( !(pCtx->cr0 & X86_CR0_PE)
4355	\|\| pCtx->cs.Attr.n.u4Type == 3)
4356	{
4357	Assert(!pCtx->ss.Attr.n.u2Dpl);
4358	}
4359	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4360	{
4361	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4362	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4363	Assert(pCtx->ss.Attr.n.u1Present);
4364	Assert(!(pCtx->ss.Attr.u & 0xf00));
4365	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4366	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4367	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4368	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4369	\|\| (pCtx->ss.Attr.n.u1Granularity));
4370	}
4371	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4372	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4373	{
4374	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4375	Assert(pCtx->ds.Attr.n.u1Present);
4376	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4377	Assert(!(pCtx->ds.Attr.u & 0xf00));
4378	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4379	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4380	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4381	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4382	\|\| (pCtx->ds.Attr.n.u1Granularity));
4383	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4384	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4385	}
4386	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4387	{
4388	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4389	Assert(pCtx->es.Attr.n.u1Present);
4390	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4391	Assert(!(pCtx->es.Attr.u & 0xf00));
4392	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4393	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4394	\|\| !(pCtx->es.Attr.n.u1Granularity));
4395	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4396	\|\| (pCtx->es.Attr.n.u1Granularity));
4397	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4398	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4399	}
4400	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4401	{
4402	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4403	Assert(pCtx->fs.Attr.n.u1Present);
4404	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4405	Assert(!(pCtx->fs.Attr.u & 0xf00));
4406	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4407	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4408	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4409	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4410	\|\| (pCtx->fs.Attr.n.u1Granularity));
4411	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4412	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4413	}
4414	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4415	{
4416	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4417	Assert(pCtx->gs.Attr.n.u1Present);
4418	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4419	Assert(!(pCtx->gs.Attr.u & 0xf00));
4420	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4421	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4422	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4423	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4424	\|\| (pCtx->gs.Attr.n.u1Granularity));
4425	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4426	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4427	}
4428	/* 64-bit capable CPUs. */
4429	# if HC_ARCH_BITS == 64
4430	Assert(!(pCtx->cs.u64Base >> 32));
4431	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4432	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4433	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4434	# endif
4435	}
4436	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4437	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4438	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4439	{
4440	/* Real and v86 mode checks. */
4441	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4442	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4443	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4444	{
4445	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4446	}
4447	else
4448	{
4449	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4450	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4451	}
4452
4453	/* CS */
4454	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4455	Assert(pCtx->cs.u32Limit == 0xffff);
4456	Assert(u32CSAttr == 0xf3);
4457	/* SS */
4458	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4459	Assert(pCtx->ss.u32Limit == 0xffff);
4460	Assert(u32SSAttr == 0xf3);
4461	/* DS */
4462	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4463	Assert(pCtx->ds.u32Limit == 0xffff);
4464	Assert(u32DSAttr == 0xf3);
4465	/* ES */
4466	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4467	Assert(pCtx->es.u32Limit == 0xffff);
4468	Assert(u32ESAttr == 0xf3);
4469	/* FS */
4470	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4471	Assert(pCtx->fs.u32Limit == 0xffff);
4472	Assert(u32FSAttr == 0xf3);
4473	/* GS */
4474	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4475	Assert(pCtx->gs.u32Limit == 0xffff);
4476	Assert(u32GSAttr == 0xf3);
4477	/* 64-bit capable CPUs. */
4478	# if HC_ARCH_BITS == 64
4479	Assert(!(pCtx->cs.u64Base >> 32));
4480	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4481	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4482	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4483	# endif
4484	}
4485	}
4486	#endif /* VBOX_STRICT */
4487
4488
4489	/**
4490	* Writes a guest segment register into the guest-state area in the VMCS.
4491	*
4492	* @returns VBox status code.
4493	* @param pVCpu The cross context virtual CPU structure.
4494	* @param idxSel Index of the selector in the VMCS.
4495	* @param idxLimit Index of the segment limit in the VMCS.
4496	* @param idxBase Index of the segment base in the VMCS.
4497	* @param idxAccess Index of the access rights of the segment in the VMCS.
4498	* @param pSelReg Pointer to the segment selector.
4499	*
4500	* @remarks No-long-jump zone!!!
4501	*/
4502	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4503	uint32_t idxAccess, PCPUMSELREG pSelReg)
4504	{
4505	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4506	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4507	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4508	AssertRCReturn(rc, rc);
4509
4510	uint32_t u32Access = pSelReg->Attr.u;
4511	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4512	{
4513	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4514	u32Access = 0xf3;
4515	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4516	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4517	}
4518	else
4519	{
4520	/*
4521	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4522	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4523	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4524	* loaded in protected-mode have their attribute as 0.
4525	*/
4526	if (!u32Access)
4527	u32Access = X86DESCATTR_UNUSABLE;
4528	}
4529
4530	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4531	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4532	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4533
4534	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4535	AssertRCReturn(rc, rc);
4536	return rc;
4537	}
4538
4539
4540	/**
4541	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4542	* into the guest-state area in the VMCS.
4543	*
4544	* @returns VBox status code.
4545	* @param pVCpu The cross context virtual CPU structure.
4546	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4547	* out-of-sync. Make sure to update the required fields
4548	* before using them.
4549	*
4550	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4551	* @remarks No-long-jump zone!!!
4552	*/
4553	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4554	{
4555	int rc = VERR_INTERNAL_ERROR_5;
4556	PVM pVM = pVCpu->CTX_SUFF(pVM);
4557
4558	/*
4559	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4560	*/
4561	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4562	{
4563	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4564	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4565	{
4566	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4567	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4568	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4569	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4570	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4571	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4572	}
4573
4574	#ifdef VBOX_WITH_REM
4575	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4576	{
4577	Assert(pVM->hm.s.vmx.pRealModeTSS);
4578	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4579	if ( pVCpu->hm.s.vmx.fWasInRealMode
4580	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4581	{
4582	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4583	in real-mode (e.g. OpenBSD 4.0) */
4584	REMFlushTBs(pVM);
4585	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4586	pVCpu->hm.s.vmx.fWasInRealMode = false;
4587	}
4588	}
4589	#endif
4590	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4591	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4592	AssertRCReturn(rc, rc);
4593	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4594	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4595	AssertRCReturn(rc, rc);
4596	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4597	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4598	AssertRCReturn(rc, rc);
4599	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4600	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4601	AssertRCReturn(rc, rc);
4602	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4603	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4604	AssertRCReturn(rc, rc);
4605	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4606	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4607	AssertRCReturn(rc, rc);
4608
4609	#ifdef VBOX_STRICT
4610	/* Validate. */
4611	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4612	#endif
4613
4614	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4615	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4616	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4617	}
4618
4619	/*
4620	* Guest TR.
4621	*/
4622	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4623	{
4624	/*
4625	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4626	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4627	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4628	*/
4629	uint16_t u16Sel = 0;
4630	uint32_t u32Limit = 0;
4631	uint64_t u64Base = 0;
4632	uint32_t u32AccessRights = 0;
4633
4634	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4635	{
4636	u16Sel = pMixedCtx->tr.Sel;
4637	u32Limit = pMixedCtx->tr.u32Limit;
4638	u64Base = pMixedCtx->tr.u64Base;
4639	u32AccessRights = pMixedCtx->tr.Attr.u;
4640	}
4641	else
4642	{
4643	Assert(pVM->hm.s.vmx.pRealModeTSS);
4644	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4645
4646	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4647	RTGCPHYS GCPhys;
4648	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4649	AssertRCReturn(rc, rc);
4650
4651	X86DESCATTR DescAttr;
4652	DescAttr.u = 0;
4653	DescAttr.n.u1Present = 1;
4654	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4655
4656	u16Sel = 0;
4657	u32Limit = HM_VTX_TSS_SIZE;
4658	u64Base = GCPhys; /* in real-mode phys = virt. */
4659	u32AccessRights = DescAttr.u;
4660	}
4661
4662	/* Validate. */
4663	Assert(!(u16Sel & RT_BIT(2)));
4664	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4665	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4666	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4667	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4668	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4669	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4670	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4671	Assert( (u32Limit & 0xfff) == 0xfff
4672	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4673	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4674	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4675
4676	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4677	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4678	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4679	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4680	AssertRCReturn(rc, rc);
4681
4682	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4683	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4684	}
4685
4686	/*
4687	* Guest GDTR.
4688	*/
4689	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4690	{
4691	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4692	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4693	AssertRCReturn(rc, rc);
4694
4695	/* Validate. */
4696	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4697
4698	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4699	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4700	}
4701
4702	/*
4703	* Guest LDTR.
4704	*/
4705	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4706	{
4707	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4708	uint32_t u32Access = 0;
4709	if (!pMixedCtx->ldtr.Attr.u)
4710	u32Access = X86DESCATTR_UNUSABLE;
4711	else
4712	u32Access = pMixedCtx->ldtr.Attr.u;
4713
4714	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4715	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4716	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4717	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4718	AssertRCReturn(rc, rc);
4719
4720	/* Validate. */
4721	if (!(u32Access & X86DESCATTR_UNUSABLE))
4722	{
4723	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4724	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4725	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4726	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4727	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4728	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4729	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4730	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4731	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4732	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4733	}
4734
4735	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4736	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4737	}
4738
4739	/*
4740	* Guest IDTR.
4741	*/
4742	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4743	{
4744	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4745	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4746	AssertRCReturn(rc, rc);
4747
4748	/* Validate. */
4749	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4750
4751	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4752	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4753	}
4754
4755	return VINF_SUCCESS;
4756	}
4757
4758
4759	/**
4760	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4761	* areas.
4762	*
4763	* These MSRs will automatically be loaded to the host CPU on every successful
4764	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4765	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4766	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4767	*
4768	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4769	*
4770	* @returns VBox status code.
4771	* @param pVCpu The cross context virtual CPU structure.
4772	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4773	* out-of-sync. Make sure to update the required fields
4774	* before using them.
4775	*
4776	* @remarks No-long-jump zone!!!
4777	*/
4778	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4779	{
4780	AssertPtr(pVCpu);
4781	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4782
4783	/*
4784	* MSRs that we use the auto-load/store MSR area in the VMCS.
4785	*/
4786	PVM pVM = pVCpu->CTX_SUFF(pVM);
4787	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4788	{
4789	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4790	#if HC_ARCH_BITS == 32
4791	if (pVM->hm.s.fAllow64BitGuests)
4792	{
4793	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4794	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4795	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4796	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4797	AssertRCReturn(rc, rc);
4798	# ifdef LOG_ENABLED
4799	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4800	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4801	{
4802	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4803	pMsr->u64Value));
4804	}
4805	# endif
4806	}
4807	#endif
4808	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4809	}
4810
4811	/*
4812	* Guest Sysenter MSRs.
4813	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4814	* VM-exits on WRMSRs for these MSRs.
4815	*/
4816	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4817	{
4818	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4819	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4820	}
4821
4822	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4823	{
4824	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4825	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4826	}
4827
4828	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4829	{
4830	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4831	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4832	}
4833
4834	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4835	{
4836	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4837	{
4838	/*
4839	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4840	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4841	*/
4842	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4843	{
4844	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4845	AssertRCReturn(rc,rc);
4846	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4847	}
4848	else
4849	{
4850	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4851	NULL /* pfAddedAndUpdated */);
4852	AssertRCReturn(rc, rc);
4853
4854	/* We need to intercept reads too, see @bugref{7386#c16}. */
4855	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4856	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4857	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4858	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4859	}
4860	}
4861	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4862	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4863	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4864	}
4865
4866	return VINF_SUCCESS;
4867	}
4868
4869
4870	/**
4871	* Loads the guest activity state into the guest-state area in the VMCS.
4872	*
4873	* @returns VBox status code.
4874	* @param pVCpu The cross context virtual CPU structure.
4875	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4876	* out-of-sync. Make sure to update the required fields
4877	* before using them.
4878	*
4879	* @remarks No-long-jump zone!!!
4880	*/
4881	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4882	{
4883	NOREF(pMixedCtx);
4884	/** @todo See if we can make use of other states, e.g.
4885	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4886	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4887	{
4888	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4889	AssertRCReturn(rc, rc);
4890
4891	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4892	}
4893	return VINF_SUCCESS;
4894	}
4895
4896
4897	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4898	/**
4899	* Check if guest state allows safe use of 32-bit switcher again.
4900	*
4901	* Segment bases and protected mode structures must be 32-bit addressable
4902	* because the 32-bit switcher will ignore high dword when writing these VMCS
4903	* fields. See @bugref{8432} for details.
4904	*
4905	* @returns true if safe, false if must continue to use the 64-bit switcher.
4906	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4907	* out-of-sync. Make sure to update the required fields
4908	* before using them.
4909	*
4910	* @remarks No-long-jump zone!!!
4911	*/
4912	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4913	{
4914	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4915	return false;
4916	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4917	return false;
4918	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4919	return false;
4920	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4921	return false;
4922	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4923	return false;
4924	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4925	return false;
4926	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4927	return false;
4928	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4929	return false;
4930	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4931	return false;
4932	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4933	return false;
4934	/* All good, bases are 32-bit. */
4935	return true;
4936	}
4937	#endif
4938
4939
4940	/**
4941	* Sets up the appropriate function to run guest code.
4942	*
4943	* @returns VBox status code.
4944	* @param pVCpu The cross context virtual CPU structure.
4945	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4946	* out-of-sync. Make sure to update the required fields
4947	* before using them.
4948	*
4949	* @remarks No-long-jump zone!!!
4950	*/
4951	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4952	{
4953	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4954	{
4955	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4956	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4957	#endif
4958	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4959	#if HC_ARCH_BITS == 32
4960	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4961	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4962	{
4963	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4964	{
4965	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4966	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4967	\| HM_CHANGED_VMX_ENTRY_CTLS
4968	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4969	}
4970	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4971
4972	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4973	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4974	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4975	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4976	}
4977	#else
4978	/* 64-bit host. */
4979	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4980	#endif
4981	}
4982	else
4983	{
4984	/* Guest is not in long mode, use the 32-bit handler. */
4985	#if HC_ARCH_BITS == 32
4986	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4987	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4988	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4989	{
4990	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4991	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4992	\| HM_CHANGED_VMX_ENTRY_CTLS
4993	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4994	}
4995	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4996	/*
4997	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4998	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4999	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
5000	* the much faster 32-bit switcher again.
5001	*/
5002	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
5003	{
5004	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
5005	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
5006	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5007	}
5008	else
5009	{
5010	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
5011	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
5012	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
5013	{
5014	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5015	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5016	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5017	\| HM_CHANGED_VMX_ENTRY_CTLS
5018	\| HM_CHANGED_VMX_EXIT_CTLS
5019	\| HM_CHANGED_HOST_CONTEXT);
5020	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5021	}
5022	}
5023	# else
5024	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5025	# endif
5026	#else
5027	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5028	#endif
5029	}
5030	Assert(pVCpu->hm.s.vmx.pfnStartVM);
5031	return VINF_SUCCESS;
5032	}
5033
5034
5035	/**
5036	* Wrapper for running the guest code in VT-x.
5037	*
5038	* @returns VBox status code, no informational status codes.
5039	* @param pVM The cross context VM structure.
5040	* @param pVCpu The cross context virtual CPU structure.
5041	* @param pCtx Pointer to the guest-CPU context.
5042	*
5043	* @remarks No-long-jump zone!!!
5044	*/
5045	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5046	{
5047	/*
5048	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5049	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5050	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5051	*/
5052	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5053	/** @todo Add stats for resume vs launch. */
5054	#ifdef VBOX_WITH_KERNEL_USING_XMM
5055	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5056	#else
5057	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5058	#endif
5059	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5060	return rc;
5061	}
5062
5063
5064	/**
5065	* Reports world-switch error and dumps some useful debug info.
5066	*
5067	* @param pVM The cross context VM structure.
5068	* @param pVCpu The cross context virtual CPU structure.
5069	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5070	* @param pCtx Pointer to the guest-CPU context.
5071	* @param pVmxTransient Pointer to the VMX transient structure (only
5072	* exitReason updated).
5073	*/
5074	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5075	{
5076	Assert(pVM);
5077	Assert(pVCpu);
5078	Assert(pCtx);
5079	Assert(pVmxTransient);
5080	HMVMX_ASSERT_PREEMPT_SAFE();
5081
5082	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5083	switch (rcVMRun)
5084	{
5085	case VERR_VMX_INVALID_VMXON_PTR:
5086	AssertFailed();
5087	break;
5088	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5089	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5090	{
5091	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5092	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5093	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5094	AssertRC(rc);
5095
5096	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5097	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5098	Cannot do it here as we may have been long preempted. */
5099
5100	#ifdef VBOX_STRICT
5101	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5102	pVmxTransient->uExitReason));
5103	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5104	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5105	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5106	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5107	else
5108	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5109	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5110	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5111
5112	/* VMX control bits. */
5113	uint32_t u32Val;
5114	uint64_t u64Val;
5115	RTHCUINTREG uHCReg;
5116	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5117	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5119	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5120	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5121	{
5122	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5123	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5124	}
5125	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5126	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5128	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5129	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5130	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5131	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5132	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5133	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5134	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5135	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5136	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5137	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5138	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5139	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5140	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5141	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5142	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5143	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5144	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5145	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5146	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5147	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5148	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5149	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5150	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5151	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5152	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5153	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5154	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5155	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5156	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5157	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5158	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5159	if (pVM->hm.s.fNestedPaging)
5160	{
5161	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5162	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5163	}
5164
5165	/* Guest bits. */
5166	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5167	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5168	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5169	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5170	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5171	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5172	if (pVM->hm.s.vmx.fVpid)
5173	{
5174	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5175	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5176	}
5177
5178	/* Host bits. */
5179	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5180	Log4(("Host CR0 %#RHr\n", uHCReg));
5181	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5182	Log4(("Host CR3 %#RHr\n", uHCReg));
5183	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5184	Log4(("Host CR4 %#RHr\n", uHCReg));
5185
5186	RTGDTR HostGdtr;
5187	PCX86DESCHC pDesc;
5188	ASMGetGDTR(&HostGdtr);
5189	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5190	Log4(("Host CS %#08x\n", u32Val));
5191	if (u32Val < HostGdtr.cbGdt)
5192	{
5193	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5194	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5195	}
5196
5197	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5198	Log4(("Host DS %#08x\n", u32Val));
5199	if (u32Val < HostGdtr.cbGdt)
5200	{
5201	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5202	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5203	}
5204
5205	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5206	Log4(("Host ES %#08x\n", u32Val));
5207	if (u32Val < HostGdtr.cbGdt)
5208	{
5209	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5210	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5211	}
5212
5213	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5214	Log4(("Host FS %#08x\n", u32Val));
5215	if (u32Val < HostGdtr.cbGdt)
5216	{
5217	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5218	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5219	}
5220
5221	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5222	Log4(("Host GS %#08x\n", u32Val));
5223	if (u32Val < HostGdtr.cbGdt)
5224	{
5225	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5226	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5227	}
5228
5229	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5230	Log4(("Host SS %#08x\n", u32Val));
5231	if (u32Val < HostGdtr.cbGdt)
5232	{
5233	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5234	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5235	}
5236
5237	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5238	Log4(("Host TR %#08x\n", u32Val));
5239	if (u32Val < HostGdtr.cbGdt)
5240	{
5241	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5242	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5243	}
5244
5245	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5246	Log4(("Host TR Base %#RHv\n", uHCReg));
5247	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5248	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5249	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5250	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5251	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5252	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5253	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5254	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5255	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5256	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5257	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5258	Log4(("Host RSP %#RHv\n", uHCReg));
5259	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5260	Log4(("Host RIP %#RHv\n", uHCReg));
5261	# if HC_ARCH_BITS == 64
5262	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5263	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5264	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5265	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5266	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5267	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5268	# endif
5269	#endif /* VBOX_STRICT */
5270	break;
5271	}
5272
5273	default:
5274	/* Impossible */
5275	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5276	break;
5277	}
5278	NOREF(pVM); NOREF(pCtx);
5279	}
5280
5281
5282	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5283	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5284	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5285	#endif
5286	#ifdef VBOX_STRICT
5287	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5288	{
5289	switch (idxField)
5290	{
5291	case VMX_VMCS_GUEST_RIP:
5292	case VMX_VMCS_GUEST_RSP:
5293	case VMX_VMCS_GUEST_SYSENTER_EIP:
5294	case VMX_VMCS_GUEST_SYSENTER_ESP:
5295	case VMX_VMCS_GUEST_GDTR_BASE:
5296	case VMX_VMCS_GUEST_IDTR_BASE:
5297	case VMX_VMCS_GUEST_CS_BASE:
5298	case VMX_VMCS_GUEST_DS_BASE:
5299	case VMX_VMCS_GUEST_ES_BASE:
5300	case VMX_VMCS_GUEST_FS_BASE:
5301	case VMX_VMCS_GUEST_GS_BASE:
5302	case VMX_VMCS_GUEST_SS_BASE:
5303	case VMX_VMCS_GUEST_LDTR_BASE:
5304	case VMX_VMCS_GUEST_TR_BASE:
5305	case VMX_VMCS_GUEST_CR3:
5306	return true;
5307	}
5308	return false;
5309	}
5310
5311	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5312	{
5313	switch (idxField)
5314	{
5315	/* Read-only fields. */
5316	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5317	return true;
5318	}
5319	/* Remaining readable fields should also be writable. */
5320	return hmR0VmxIsValidWriteField(idxField);
5321	}
5322	#endif /* VBOX_STRICT */
5323
5324
5325	/**
5326	* Executes the specified handler in 64-bit mode.
5327	*
5328	* @returns VBox status code (no informational status codes).
5329	* @param pVM The cross context VM structure.
5330	* @param pVCpu The cross context virtual CPU structure.
5331	* @param pCtx Pointer to the guest CPU context.
5332	* @param enmOp The operation to perform.
5333	* @param cParams Number of parameters.
5334	* @param paParam Array of 32-bit parameters.
5335	*/
5336	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5337	uint32_t cParams, uint32_t *paParam)
5338	{
5339	NOREF(pCtx);
5340
5341	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5342	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5343	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5344	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5345
5346	#ifdef VBOX_STRICT
5347	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5348	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5349
5350	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5351	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5352	#endif
5353
5354	/* Disable interrupts. */
5355	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5356
5357	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5358	RTCPUID idHostCpu = RTMpCpuId();
5359	CPUMR0SetLApic(pVCpu, idHostCpu);
5360	#endif
5361
5362	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5363	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5364
5365	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5366	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5367	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5368
5369	/* Leave VMX Root Mode. */
5370	VMXDisable();
5371
5372	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5373
5374	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5375	CPUMSetHyperEIP(pVCpu, enmOp);
5376	for (int i = (int)cParams - 1; i >= 0; i--)
5377	CPUMPushHyper(pVCpu, paParam[i]);
5378
5379	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5380
5381	/* Call the switcher. */
5382	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5383	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5384
5385	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5386	/* Make sure the VMX instructions don't cause #UD faults. */
5387	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5388
5389	/* Re-enter VMX Root Mode */
5390	int rc2 = VMXEnable(HCPhysCpuPage);
5391	if (RT_FAILURE(rc2))
5392	{
5393	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5394	ASMSetFlags(fOldEFlags);
5395	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5396	return rc2;
5397	}
5398
5399	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5400	AssertRC(rc2);
5401	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5402	Assert(!(ASMGetFlags() & X86_EFL_IF));
5403	ASMSetFlags(fOldEFlags);
5404	return rc;
5405	}
5406
5407
5408	/**
5409	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5410	* supporting 64-bit guests.
5411	*
5412	* @returns VBox status code.
5413	* @param fResume Whether to VMLAUNCH or VMRESUME.
5414	* @param pCtx Pointer to the guest-CPU context.
5415	* @param pCache Pointer to the VMCS cache.
5416	* @param pVM The cross context VM structure.
5417	* @param pVCpu The cross context virtual CPU structure.
5418	*/
5419	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5420	{
5421	NOREF(fResume);
5422
5423	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5424	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5425
5426	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5427	pCache->uPos = 1;
5428	pCache->interPD = PGMGetInterPaeCR3(pVM);
5429	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5430	#endif
5431
5432	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5433	pCache->TestIn.HCPhysCpuPage = 0;
5434	pCache->TestIn.HCPhysVmcs = 0;
5435	pCache->TestIn.pCache = 0;
5436	pCache->TestOut.HCPhysVmcs = 0;
5437	pCache->TestOut.pCache = 0;
5438	pCache->TestOut.pCtx = 0;
5439	pCache->TestOut.eflags = 0;
5440	#else
5441	NOREF(pCache);
5442	#endif
5443
5444	uint32_t aParam[10];
5445	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5446	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5447	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5448	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5449	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5450	aParam[5] = 0;
5451	aParam[6] = VM_RC_ADDR(pVM, pVM);
5452	aParam[7] = 0;
5453	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5454	aParam[9] = 0;
5455
5456	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5457	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5458	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5459	#endif
5460	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5461
5462	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5463	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5464	Assert(pCtx->dr[4] == 10);
5465	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5466	#endif
5467
5468	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5469	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5470	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5471	pVCpu->hm.s.vmx.HCPhysVmcs));
5472	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5473	pCache->TestOut.HCPhysVmcs));
5474	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5475	pCache->TestOut.pCache));
5476	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5477	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5478	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5479	pCache->TestOut.pCtx));
5480	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5481	#endif
5482	return rc;
5483	}
5484
5485
5486	/**
5487	* Initialize the VMCS-Read cache.
5488	*
5489	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5490	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5491	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5492	* (those that have a 32-bit FULL & HIGH part).
5493	*
5494	* @returns VBox status code.
5495	* @param pVM The cross context VM structure.
5496	* @param pVCpu The cross context virtual CPU structure.
5497	*/
5498	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5499	{
5500	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5501	{ \
5502	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5503	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5504	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5505	++cReadFields; \
5506	}
5507
5508	AssertPtr(pVM);
5509	AssertPtr(pVCpu);
5510	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5511	uint32_t cReadFields = 0;
5512
5513	/*
5514	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5515	* and serve to indicate exceptions to the rules.
5516	*/
5517
5518	/* Guest-natural selector base fields. */
5519	#if 0
5520	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5523	#endif
5524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5530	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5531	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5532	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5533	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5534	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5535	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5536	#if 0
5537	/* Unused natural width guest-state fields. */
5538	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5539	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5540	#endif
5541	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5542	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5543
5544	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5545	#if 0
5546	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5547	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5548	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5549	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5550	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5551	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5552	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5553	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5554	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5555	#endif
5556
5557	/* Natural width guest-state fields. */
5558	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5559	#if 0
5560	/* Currently unused field. */
5561	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5562	#endif
5563
5564	if (pVM->hm.s.fNestedPaging)
5565	{
5566	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5567	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5568	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5569	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5570	}
5571	else
5572	{
5573	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5574	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5575	}
5576
5577	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5578	return VINF_SUCCESS;
5579	}
5580
5581
5582	/**
5583	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5584	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5585	* darwin, running 64-bit guests).
5586	*
5587	* @returns VBox status code.
5588	* @param pVCpu The cross context virtual CPU structure.
5589	* @param idxField The VMCS field encoding.
5590	* @param u64Val 16, 32 or 64-bit value.
5591	*/
5592	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5593	{
5594	int rc;
5595	switch (idxField)
5596	{
5597	/*
5598	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5599	*/
5600	/* 64-bit Control fields. */
5601	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5602	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5603	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5604	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5605	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5606	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5607	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5608	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5609	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5610	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5611	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5612	case VMX_VMCS64_CTRL_EPTP_FULL:
5613	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5614	/* 64-bit Guest-state fields. */
5615	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5616	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5617	case VMX_VMCS64_GUEST_PAT_FULL:
5618	case VMX_VMCS64_GUEST_EFER_FULL:
5619	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5620	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5621	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5622	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5623	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5624	/* 64-bit Host-state fields. */
5625	case VMX_VMCS64_HOST_PAT_FULL:
5626	case VMX_VMCS64_HOST_EFER_FULL:
5627	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5628	{
5629	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5630	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5631	break;
5632	}
5633
5634	/*
5635	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5636	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5637	*/
5638	/* Natural-width Guest-state fields. */
5639	case VMX_VMCS_GUEST_CR3:
5640	case VMX_VMCS_GUEST_ES_BASE:
5641	case VMX_VMCS_GUEST_CS_BASE:
5642	case VMX_VMCS_GUEST_SS_BASE:
5643	case VMX_VMCS_GUEST_DS_BASE:
5644	case VMX_VMCS_GUEST_FS_BASE:
5645	case VMX_VMCS_GUEST_GS_BASE:
5646	case VMX_VMCS_GUEST_LDTR_BASE:
5647	case VMX_VMCS_GUEST_TR_BASE:
5648	case VMX_VMCS_GUEST_GDTR_BASE:
5649	case VMX_VMCS_GUEST_IDTR_BASE:
5650	case VMX_VMCS_GUEST_RSP:
5651	case VMX_VMCS_GUEST_RIP:
5652	case VMX_VMCS_GUEST_SYSENTER_ESP:
5653	case VMX_VMCS_GUEST_SYSENTER_EIP:
5654	{
5655	if (!(RT_HI_U32(u64Val)))
5656	{
5657	/* If this field is 64-bit, VT-x will zero out the top bits. */
5658	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5659	}
5660	else
5661	{
5662	/* Assert that only the 32->64 switcher case should ever come here. */
5663	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5664	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5665	}
5666	break;
5667	}
5668
5669	default:
5670	{
5671	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5672	rc = VERR_INVALID_PARAMETER;
5673	break;
5674	}
5675	}
5676	AssertRCReturn(rc, rc);
5677	return rc;
5678	}
5679
5680
5681	/**
5682	* Queue up a VMWRITE by using the VMCS write cache.
5683	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5684	*
5685	* @param pVCpu The cross context virtual CPU structure.
5686	* @param idxField The VMCS field encoding.
5687	* @param u64Val 16, 32 or 64-bit value.
5688	*/
5689	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5690	{
5691	AssertPtr(pVCpu);
5692	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5693
5694	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5695	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5696
5697	/* Make sure there are no duplicates. */
5698	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5699	{
5700	if (pCache->Write.aField[i] == idxField)
5701	{
5702	pCache->Write.aFieldVal[i] = u64Val;
5703	return VINF_SUCCESS;
5704	}
5705	}
5706
5707	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5708	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5709	pCache->Write.cValidEntries++;
5710	return VINF_SUCCESS;
5711	}
5712	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5713
5714
5715	/**
5716	* Sets up the usage of TSC-offsetting and updates the VMCS.
5717	*
5718	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5719	* VMX preemption timer.
5720	*
5721	* @returns VBox status code.
5722	* @param pVM The cross context VM structure.
5723	* @param pVCpu The cross context virtual CPU structure.
5724	*
5725	* @remarks No-long-jump zone!!!
5726	*/
5727	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5728	{
5729	int rc;
5730	bool fOffsettedTsc;
5731	bool fParavirtTsc;
5732	if (pVM->hm.s.vmx.fUsePreemptTimer)
5733	{
5734	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5735	&fOffsettedTsc, &fParavirtTsc);
5736
5737	/* Make sure the returned values have sane upper and lower boundaries. */
5738	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5739	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5740	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5741	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5742
5743	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5744	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5745	}
5746	else
5747	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5748
5749	/** @todo later optimize this to be done elsewhere and not before every
5750	* VM-entry. */
5751	if (fParavirtTsc)
5752	{
5753	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5754	information before every VM-entry, hence disable it for performance sake. */
5755	#if 0
5756	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5757	AssertRC(rc);
5758	#endif
5759	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5760	}
5761
5762	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5763	{
5764	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5765	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5766
5767	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5768	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5769	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5770	}
5771	else
5772	{
5773	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5774	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5775	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5776	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5777	}
5778	}
5779
5780
5781	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5782	/**
5783	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5784	* VM-exit interruption info type.
5785	*
5786	* @returns The IEM exception flags.
5787	* @param uVector The event vector.
5788	* @param uVmxVectorType The VMX event type.
5789	*
5790	* @remarks This function currently only constructs flags required for
5791	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5792	* and CR2 aspects of an exception are not included).
5793	*/
5794	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5795	{
5796	uint32_t fIemXcptFlags;
5797	switch (uVmxVectorType)
5798	{
5799	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5800	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5801	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5802	break;
5803
5804	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5805	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5806	break;
5807
5808	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5809	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5810	break;
5811
5812	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5813	{
5814	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5815	if (uVector == X86_XCPT_BP)
5816	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5817	else if (uVector == X86_XCPT_OF)
5818	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5819	else
5820	{
5821	fIemXcptFlags = 0;
5822	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5823	}
5824	break;
5825	}
5826
5827	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5828	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5829	break;
5830
5831	default:
5832	fIemXcptFlags = 0;
5833	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5834	break;
5835	}
5836	return fIemXcptFlags;
5837	}
5838
5839	#else
5840	/**
5841	* Determines if an exception is a contributory exception.
5842	*
5843	* Contributory exceptions are ones which can cause double-faults unless the
5844	* original exception was a benign exception. Page-fault is intentionally not
5845	* included here as it's a conditional contributory exception.
5846	*
5847	* @returns true if the exception is contributory, false otherwise.
5848	* @param uVector The exception vector.
5849	*/
5850	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5851	{
5852	switch (uVector)
5853	{
5854	case X86_XCPT_GP:
5855	case X86_XCPT_SS:
5856	case X86_XCPT_NP:
5857	case X86_XCPT_TS:
5858	case X86_XCPT_DE:
5859	return true;
5860	default:
5861	break;
5862	}
5863	return false;
5864	}
5865	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5866
5867
5868	/**
5869	* Sets an event as a pending event to be injected into the guest.
5870	*
5871	* @param pVCpu The cross context virtual CPU structure.
5872	* @param u32IntInfo The VM-entry interruption-information field.
5873	* @param cbInstr The VM-entry instruction length in bytes (for software
5874	* interrupts, exceptions and privileged software
5875	* exceptions).
5876	* @param u32ErrCode The VM-entry exception error code.
5877	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5878	* page-fault.
5879	*
5880	* @remarks Statistics counter assumes this is a guest event being injected or
5881	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5882	* always incremented.
5883	*/
5884	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5885	RTGCUINTPTR GCPtrFaultAddress)
5886	{
5887	Assert(!pVCpu->hm.s.Event.fPending);
5888	pVCpu->hm.s.Event.fPending = true;
5889	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5890	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5891	pVCpu->hm.s.Event.cbInstr = cbInstr;
5892	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5893	}
5894
5895
5896	/**
5897	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5898	*
5899	* @param pVCpu The cross context virtual CPU structure.
5900	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5901	* out-of-sync. Make sure to update the required fields
5902	* before using them.
5903	*/
5904	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5905	{
5906	NOREF(pMixedCtx);
5907	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5908	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5909	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5910	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5911	}
5912
5913
5914	/**
5915	* Handle a condition that occurred while delivering an event through the guest
5916	* IDT.
5917	*
5918	* @returns Strict VBox status code (i.e. informational status codes too).
5919	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5920	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5921	* to continue execution of the guest which will delivery the \#DF.
5922	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5923	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5924	*
5925	* @param pVCpu The cross context virtual CPU structure.
5926	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5927	* out-of-sync. Make sure to update the required fields
5928	* before using them.
5929	* @param pVmxTransient Pointer to the VMX transient structure.
5930	*
5931	* @remarks No-long-jump zone!!!
5932	*/
5933	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5934	{
5935	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5936
5937	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5938	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5939
5940	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5941	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5942	{
5943	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5944	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5945	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5946	/*
5947	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5948	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5949	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5950	* exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5951	* currently has no way of storing that these exceptions were caused by these instructions
5952	* (ICEBP's #DB poses the problem).
5953	*/
5954	IEMXCPTRAISE enmRaise;
5955	IEMXCPTRAISEINFO fRaiseInfo;
5956	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5957	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5958	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5959	{
5960	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5961	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5962	}
5963	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5964	{
5965	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5966	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5967	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5968	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5969	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5970	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5971	uExitVectorType), VERR_VMX_IPE_5);
5972	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5973
5974	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5975	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5976	{
5977	pVmxTransient->fVectoringPF = true;
5978	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5979	}
5980	}
5981	else
5982	{
5983	/*
5984	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5985	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5986	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5987	*/
5988	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5989	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5990	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5991	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5992	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5993	}
5994
5995	/*
5996	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5997	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5998	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5999	* subsequent VM-entry would fail.
6000	*
6001	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6002	*/
6003	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6004	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6005	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6006	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6007	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6008	{
6009	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6010	}
6011
6012	switch (enmRaise)
6013	{
6014	case IEMXCPTRAISE_CURRENT_XCPT:
6015	{
6016	Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
6017	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6018	Assert(rcStrict == VINF_SUCCESS);
6019	break;
6020	}
6021
6022	case IEMXCPTRAISE_PREV_EVENT:
6023	{
6024	uint32_t u32ErrCode;
6025	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6026	{
6027	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6028	AssertRCReturn(rc2, rc2);
6029	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6030	}
6031	else
6032	u32ErrCode = 0;
6033
6034	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6035	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6036	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6037	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6038
6039	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6040	pVCpu->hm.s.Event.u32ErrCode));
6041	Assert(rcStrict == VINF_SUCCESS);
6042	break;
6043	}
6044
6045	case IEMXCPTRAISE_REEXEC_INSTR:
6046	Assert(rcStrict == VINF_SUCCESS);
6047	break;
6048
6049	case IEMXCPTRAISE_DOUBLE_FAULT:
6050	{
6051	/*
6052	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6053	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6054	*/
6055	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6056	{
6057	pVmxTransient->fVectoringDoublePF = true;
6058	Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6059	pMixedCtx->cr2));
6060	rcStrict = VINF_SUCCESS;
6061	}
6062	else
6063	{
6064	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6065	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6066	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6067	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6068	rcStrict = VINF_HM_DOUBLE_FAULT;
6069	}
6070	break;
6071	}
6072
6073	case IEMXCPTRAISE_TRIPLE_FAULT:
6074	{
6075	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6076	uExitVector));
6077	rcStrict = VINF_EM_RESET;
6078	break;
6079	}
6080
6081	case IEMXCPTRAISE_CPU_HANG:
6082	{
6083	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6084	rcStrict = VERR_EM_GUEST_CPU_HANG;
6085	break;
6086	}
6087
6088	default:
6089	{
6090	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6091	rcStrict = VERR_VMX_IPE_2;
6092	break;
6093	}
6094	}
6095	#else
6096	typedef enum
6097	{
6098	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6099	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6100	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6101	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6102	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6103	} VMXREFLECTXCPT;
6104
6105	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6106	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6107	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6108	{
6109	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6110	{
6111	enmReflect = VMXREFLECTXCPT_XCPT;
6112	#ifdef VBOX_STRICT
6113	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6114	&& uExitVector == X86_XCPT_PF)
6115	{
6116	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6117	}
6118	#endif
6119	if ( uExitVector == X86_XCPT_PF
6120	&& uIdtVector == X86_XCPT_PF)
6121	{
6122	pVmxTransient->fVectoringDoublePF = true;
6123	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6124	}
6125	else if ( uExitVector == X86_XCPT_AC
6126	&& uIdtVector == X86_XCPT_AC)
6127	{
6128	enmReflect = VMXREFLECTXCPT_HANG;
6129	Log4(("IDT: Nested #AC - Bad guest\n"));
6130	}
6131	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6132	&& hmR0VmxIsContributoryXcpt(uExitVector)
6133	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6134	\|\| uIdtVector == X86_XCPT_PF))
6135	{
6136	enmReflect = VMXREFLECTXCPT_DF;
6137	}
6138	else if (uIdtVector == X86_XCPT_DF)
6139	enmReflect = VMXREFLECTXCPT_TF;
6140	}
6141	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6142	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6143	{
6144	/*
6145	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6146	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6147	*/
6148	enmReflect = VMXREFLECTXCPT_XCPT;
6149
6150	if (uExitVector == X86_XCPT_PF)
6151	{
6152	pVmxTransient->fVectoringPF = true;
6153	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6154	}
6155	}
6156	}
6157	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6158	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6159	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6160	{
6161	/*
6162	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6163	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6164	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6165	*/
6166	enmReflect = VMXREFLECTXCPT_XCPT;
6167	}
6168
6169	/*
6170	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6171	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6172	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6173	*
6174	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6175	*/
6176	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6177	&& enmReflect == VMXREFLECTXCPT_XCPT
6178	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6179	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6180	{
6181	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6182	}
6183
6184	switch (enmReflect)
6185	{
6186	case VMXREFLECTXCPT_XCPT:
6187	{
6188	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6189	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6190	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6191
6192	uint32_t u32ErrCode = 0;
6193	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6194	{
6195	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6196	AssertRCReturn(rc2, rc2);
6197	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6198	}
6199
6200	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6201	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6202	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6203	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6204	rcStrict = VINF_SUCCESS;
6205	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6206	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6207
6208	break;
6209	}
6210
6211	case VMXREFLECTXCPT_DF:
6212	{
6213	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6214	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6215	rcStrict = VINF_HM_DOUBLE_FAULT;
6216	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6217	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6218
6219	break;
6220	}
6221
6222	case VMXREFLECTXCPT_TF:
6223	{
6224	rcStrict = VINF_EM_RESET;
6225	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6226	uExitVector));
6227	break;
6228	}
6229
6230	case VMXREFLECTXCPT_HANG:
6231	{
6232	rcStrict = VERR_EM_GUEST_CPU_HANG;
6233	break;
6234	}
6235
6236	default:
6237	Assert(rcStrict == VINF_SUCCESS);
6238	break;
6239	}
6240	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6241	}
6242	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6243	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6244	&& uExitVector != X86_XCPT_DF
6245	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6246	{
6247	/*
6248	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6249	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6250	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6251	*/
6252	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6253	{
6254	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6255	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6256	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6257	}
6258	}
6259
6260	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6261	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6262	return rcStrict;
6263	}
6264
6265
6266	/**
6267	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6268	*
6269	* @returns VBox status code.
6270	* @param pVCpu The cross context virtual CPU structure.
6271	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6272	* out-of-sync. Make sure to update the required fields
6273	* before using them.
6274	*
6275	* @remarks No-long-jump zone!!!
6276	*/
6277	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6278	{
6279	NOREF(pMixedCtx);
6280
6281	/*
6282	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6283	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6284	*/
6285	VMMRZCallRing3Disable(pVCpu);
6286	HM_DISABLE_PREEMPT();
6287
6288	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6289	{
6290	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6291	* and 'dbgc-init' containing:
6292	* sxe "xcpt_de"
6293	* sxe "xcpt_bp"
6294	* sxi "xcpt_gp"
6295	* sxi "xcpt_ss"
6296	* sxi "xcpt_np"
6297	*/
6298	/** @todo r=ramshankar: Should be fixed after r119291. */
6299	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6300	#endif
6301	uint32_t uVal = 0;
6302	uint32_t uShadow = 0;
6303	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6304	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6305	AssertRCReturn(rc, rc);
6306
6307	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6308	CPUMSetGuestCR0(pVCpu, uVal);
6309	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6310	}
6311
6312	HM_RESTORE_PREEMPT();
6313	VMMRZCallRing3Enable(pVCpu);
6314	return VINF_SUCCESS;
6315	}
6316
6317
6318	/**
6319	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6320	*
6321	* @returns VBox status code.
6322	* @param pVCpu The cross context virtual CPU structure.
6323	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6324	* out-of-sync. Make sure to update the required fields
6325	* before using them.
6326	*
6327	* @remarks No-long-jump zone!!!
6328	*/
6329	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6330	{
6331	NOREF(pMixedCtx);
6332
6333	int rc = VINF_SUCCESS;
6334	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6335	{
6336	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6337	uint32_t uVal = 0;
6338	uint32_t uShadow = 0;
6339	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6340	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6341	AssertRCReturn(rc, rc);
6342
6343	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6344	CPUMSetGuestCR4(pVCpu, uVal);
6345	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6346	}
6347	return rc;
6348	}
6349
6350
6351	/**
6352	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6353	*
6354	* @returns VBox status code.
6355	* @param pVCpu The cross context virtual CPU structure.
6356	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6357	* out-of-sync. Make sure to update the required fields
6358	* before using them.
6359	*
6360	* @remarks No-long-jump zone!!!
6361	*/
6362	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6363	{
6364	int rc = VINF_SUCCESS;
6365	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6366	{
6367	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6368	uint64_t u64Val = 0;
6369	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6370	AssertRCReturn(rc, rc);
6371
6372	pMixedCtx->rip = u64Val;
6373	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6374	}
6375	return rc;
6376	}
6377
6378
6379	/**
6380	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6381	*
6382	* @returns VBox status code.
6383	* @param pVCpu The cross context virtual CPU structure.
6384	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6385	* out-of-sync. Make sure to update the required fields
6386	* before using them.
6387	*
6388	* @remarks No-long-jump zone!!!
6389	*/
6390	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6391	{
6392	int rc = VINF_SUCCESS;
6393	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6394	{
6395	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6396	uint64_t u64Val = 0;
6397	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6398	AssertRCReturn(rc, rc);
6399
6400	pMixedCtx->rsp = u64Val;
6401	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6402	}
6403	return rc;
6404	}
6405
6406
6407	/**
6408	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6409	*
6410	* @returns VBox status code.
6411	* @param pVCpu The cross context virtual CPU structure.
6412	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6413	* out-of-sync. Make sure to update the required fields
6414	* before using them.
6415	*
6416	* @remarks No-long-jump zone!!!
6417	*/
6418	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6419	{
6420	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6421	{
6422	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6423	uint32_t uVal = 0;
6424	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6425	AssertRCReturn(rc, rc);
6426
6427	pMixedCtx->eflags.u32 = uVal;
6428	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6429	{
6430	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6431	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6432
6433	pMixedCtx->eflags.Bits.u1VM = 0;
6434	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6435	}
6436
6437	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6438	}
6439	return VINF_SUCCESS;
6440	}
6441
6442
6443	/**
6444	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6445	* guest-CPU context.
6446	*/
6447	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6448	{
6449	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6450	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6451	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6452	return rc;
6453	}
6454
6455
6456	/**
6457	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6458	* from the guest-state area in the VMCS.
6459	*
6460	* @param pVCpu The cross context virtual CPU structure.
6461	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6462	* out-of-sync. Make sure to update the required fields
6463	* before using them.
6464	*
6465	* @remarks No-long-jump zone!!!
6466	*/
6467	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6468	{
6469	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6470	{
6471	uint32_t uIntrState = 0;
6472	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6473	AssertRC(rc);
6474
6475	if (!uIntrState)
6476	{
6477	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6478	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6479
6480	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6481	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6482	}
6483	else
6484	{
6485	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6486	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6487	{
6488	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6489	AssertRC(rc);
6490	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6491	AssertRC(rc);
6492
6493	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6494	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6495	}
6496	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6497	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6498
6499	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6500	{
6501	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6502	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6503	}
6504	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6505	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6506	}
6507
6508	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6509	}
6510	}
6511
6512
6513	/**
6514	* Saves the guest's activity state.
6515	*
6516	* @returns VBox status code.
6517	* @param pVCpu The cross context virtual CPU structure.
6518	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6519	* out-of-sync. Make sure to update the required fields
6520	* before using them.
6521	*
6522	* @remarks No-long-jump zone!!!
6523	*/
6524	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6525	{
6526	NOREF(pMixedCtx);
6527	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6528	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6529	return VINF_SUCCESS;
6530	}
6531
6532
6533	/**
6534	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6535	* the current VMCS into the guest-CPU context.
6536	*
6537	* @returns VBox status code.
6538	* @param pVCpu The cross context virtual CPU structure.
6539	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6540	* out-of-sync. Make sure to update the required fields
6541	* before using them.
6542	*
6543	* @remarks No-long-jump zone!!!
6544	*/
6545	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6546	{
6547	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6548	{
6549	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6550	uint32_t u32Val = 0;
6551	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6552	pMixedCtx->SysEnter.cs = u32Val;
6553	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6554	}
6555
6556	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6557	{
6558	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6559	uint64_t u64Val = 0;
6560	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6561	pMixedCtx->SysEnter.eip = u64Val;
6562	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6563	}
6564	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6565	{
6566	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6567	uint64_t u64Val = 0;
6568	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6569	pMixedCtx->SysEnter.esp = u64Val;
6570	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6571	}
6572	return VINF_SUCCESS;
6573	}
6574
6575
6576	/**
6577	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6578	* the CPU back into the guest-CPU context.
6579	*
6580	* @returns VBox status code.
6581	* @param pVCpu The cross context virtual CPU structure.
6582	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6583	* out-of-sync. Make sure to update the required fields
6584	* before using them.
6585	*
6586	* @remarks No-long-jump zone!!!
6587	*/
6588	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6589	{
6590	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6591	VMMRZCallRing3Disable(pVCpu);
6592	HM_DISABLE_PREEMPT();
6593
6594	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6595	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6596	{
6597	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6598	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6599	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6600	}
6601
6602	HM_RESTORE_PREEMPT();
6603	VMMRZCallRing3Enable(pVCpu);
6604
6605	return VINF_SUCCESS;
6606	}
6607
6608
6609	/**
6610	* Saves the auto load/store'd guest MSRs from the current VMCS into
6611	* the guest-CPU context.
6612	*
6613	* @returns VBox status code.
6614	* @param pVCpu The cross context virtual CPU structure.
6615	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6616	* out-of-sync. Make sure to update the required fields
6617	* before using them.
6618	*
6619	* @remarks No-long-jump zone!!!
6620	*/
6621	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6622	{
6623	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6624	return VINF_SUCCESS;
6625
6626	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6627	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6628	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6629	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6630	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6631	{
6632	switch (pMsr->u32Msr)
6633	{
6634	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6635	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6636	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6637	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6638	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6639	case MSR_IA32_SPEC_CTRL: CPUMR0SetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6640	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6641	break;
6642
6643	default:
6644	{
6645	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6646	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6647	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6648	}
6649	}
6650	}
6651
6652	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6653	return VINF_SUCCESS;
6654	}
6655
6656
6657	/**
6658	* Saves the guest control registers from the current VMCS into the guest-CPU
6659	* context.
6660	*
6661	* @returns VBox status code.
6662	* @param pVCpu The cross context virtual CPU structure.
6663	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6664	* out-of-sync. Make sure to update the required fields
6665	* before using them.
6666	*
6667	* @remarks No-long-jump zone!!!
6668	*/
6669	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6670	{
6671	/* Guest CR0. Guest FPU. */
6672	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6673	AssertRCReturn(rc, rc);
6674
6675	/* Guest CR4. */
6676	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6677	AssertRCReturn(rc, rc);
6678
6679	/* Guest CR2 - updated always during the world-switch or in #PF. */
6680	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6681	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6682	{
6683	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6684	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6685	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6686
6687	PVM pVM = pVCpu->CTX_SUFF(pVM);
6688	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6689	\|\| ( pVM->hm.s.fNestedPaging
6690	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6691	{
6692	uint64_t u64Val = 0;
6693	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6694	if (pMixedCtx->cr3 != u64Val)
6695	{
6696	CPUMSetGuestCR3(pVCpu, u64Val);
6697	if (VMMRZCallRing3IsEnabled(pVCpu))
6698	{
6699	PGMUpdateCR3(pVCpu, u64Val);
6700	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6701	}
6702	else
6703	{
6704	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6705	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6706	}
6707	}
6708
6709	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6710	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6711	{
6712	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6713	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6714	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6715	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6716	AssertRCReturn(rc, rc);
6717
6718	if (VMMRZCallRing3IsEnabled(pVCpu))
6719	{
6720	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6721	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6722	}
6723	else
6724	{
6725	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6726	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6727	}
6728	}
6729	}
6730
6731	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6732	}
6733
6734	/*
6735	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6736	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6737	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6738	*
6739	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6740	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6741	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6742	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6743	*
6744	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6745	*/
6746	if (VMMRZCallRing3IsEnabled(pVCpu))
6747	{
6748	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6749	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6750
6751	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6752	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6753
6754	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6755	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6756	}
6757
6758	return rc;
6759	}
6760
6761
6762	/**
6763	* Reads a guest segment register from the current VMCS into the guest-CPU
6764	* context.
6765	*
6766	* @returns VBox status code.
6767	* @param pVCpu The cross context virtual CPU structure.
6768	* @param idxSel Index of the selector in the VMCS.
6769	* @param idxLimit Index of the segment limit in the VMCS.
6770	* @param idxBase Index of the segment base in the VMCS.
6771	* @param idxAccess Index of the access rights of the segment in the VMCS.
6772	* @param pSelReg Pointer to the segment selector.
6773	*
6774	* @remarks No-long-jump zone!!!
6775	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6776	* macro as that takes care of whether to read from the VMCS cache or
6777	* not.
6778	*/
6779	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6780	PCPUMSELREG pSelReg)
6781	{
6782	NOREF(pVCpu);
6783
6784	uint32_t u32Val = 0;
6785	int rc = VMXReadVmcs32(idxSel, &u32Val);
6786	AssertRCReturn(rc, rc);
6787	pSelReg->Sel = (uint16_t)u32Val;
6788	pSelReg->ValidSel = (uint16_t)u32Val;
6789	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6790
6791	rc = VMXReadVmcs32(idxLimit, &u32Val);
6792	AssertRCReturn(rc, rc);
6793	pSelReg->u32Limit = u32Val;
6794
6795	uint64_t u64Val = 0;
6796	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6797	AssertRCReturn(rc, rc);
6798	pSelReg->u64Base = u64Val;
6799
6800	rc = VMXReadVmcs32(idxAccess, &u32Val);
6801	AssertRCReturn(rc, rc);
6802	pSelReg->Attr.u = u32Val;
6803
6804	/*
6805	* If VT-x marks the segment as unusable, most other bits remain undefined:
6806	* - For CS the L, D and G bits have meaning.
6807	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6808	* - For the remaining data segments no bits are defined.
6809	*
6810	* The present bit and the unusable bit has been observed to be set at the
6811	* same time (the selector was supposed to be invalid as we started executing
6812	* a V8086 interrupt in ring-0).
6813	*
6814	* What should be important for the rest of the VBox code, is that the P bit is
6815	* cleared. Some of the other VBox code recognizes the unusable bit, but
6816	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6817	* safe side here, we'll strip off P and other bits we don't care about. If
6818	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6819	*
6820	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6821	*/
6822	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6823	{
6824	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6825
6826	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6827	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6828	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6829
6830	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6831	#ifdef DEBUG_bird
6832	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6833	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6834	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6835	#endif
6836	}
6837	return VINF_SUCCESS;
6838	}
6839
6840
6841	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6842	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6843	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6844	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6845	#else
6846	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6847	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6848	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6849	#endif
6850
6851
6852	/**
6853	* Saves the guest segment registers from the current VMCS into the guest-CPU
6854	* context.
6855	*
6856	* @returns VBox status code.
6857	* @param pVCpu The cross context virtual CPU structure.
6858	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6859	* out-of-sync. Make sure to update the required fields
6860	* before using them.
6861	*
6862	* @remarks No-long-jump zone!!!
6863	*/
6864	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6865	{
6866	/* Guest segment registers. */
6867	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6868	{
6869	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6870	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6871	AssertRCReturn(rc, rc);
6872
6873	rc = VMXLOCAL_READ_SEG(CS, cs);
6874	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6875	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6876	rc \|= VMXLOCAL_READ_SEG(ES, es);
6877	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6878	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6879	AssertRCReturn(rc, rc);
6880
6881	/* Restore segment attributes for real-on-v86 mode hack. */
6882	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6883	{
6884	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6885	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6886	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6887	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6888	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6889	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6890	}
6891	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6892	}
6893
6894	return VINF_SUCCESS;
6895	}
6896
6897
6898	/**
6899	* Saves the guest descriptor table registers and task register from the current
6900	* VMCS into the guest-CPU context.
6901	*
6902	* @returns VBox status code.
6903	* @param pVCpu The cross context virtual CPU structure.
6904	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6905	* out-of-sync. Make sure to update the required fields
6906	* before using them.
6907	*
6908	* @remarks No-long-jump zone!!!
6909	*/
6910	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6911	{
6912	int rc = VINF_SUCCESS;
6913
6914	/* Guest LDTR. */
6915	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6916	{
6917	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6918	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6919	AssertRCReturn(rc, rc);
6920	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6921	}
6922
6923	/* Guest GDTR. */
6924	uint64_t u64Val = 0;
6925	uint32_t u32Val = 0;
6926	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6927	{
6928	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6929	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6930	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6931	pMixedCtx->gdtr.pGdt = u64Val;
6932	pMixedCtx->gdtr.cbGdt = u32Val;
6933	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6934	}
6935
6936	/* Guest IDTR. */
6937	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6938	{
6939	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6940	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6941	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6942	pMixedCtx->idtr.pIdt = u64Val;
6943	pMixedCtx->idtr.cbIdt = u32Val;
6944	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6945	}
6946
6947	/* Guest TR. */
6948	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6949	{
6950	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6951	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6952	AssertRCReturn(rc, rc);
6953
6954	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6955	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6956	{
6957	rc = VMXLOCAL_READ_SEG(TR, tr);
6958	AssertRCReturn(rc, rc);
6959	}
6960	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6961	}
6962	return rc;
6963	}
6964
6965	#undef VMXLOCAL_READ_SEG
6966
6967
6968	/**
6969	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6970	* context.
6971	*
6972	* @returns VBox status code.
6973	* @param pVCpu The cross context virtual CPU structure.
6974	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6975	* out-of-sync. Make sure to update the required fields
6976	* before using them.
6977	*
6978	* @remarks No-long-jump zone!!!
6979	*/
6980	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6981	{
6982	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6983	{
6984	if (!pVCpu->hm.s.fUsingHyperDR7)
6985	{
6986	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6987	uint32_t u32Val;
6988	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6989	pMixedCtx->dr[7] = u32Val;
6990	}
6991
6992	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6993	}
6994	return VINF_SUCCESS;
6995	}
6996
6997
6998	/**
6999	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
7000	*
7001	* @returns VBox status code.
7002	* @param pVCpu The cross context virtual CPU structure.
7003	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7004	* out-of-sync. Make sure to update the required fields
7005	* before using them.
7006	*
7007	* @remarks No-long-jump zone!!!
7008	*/
7009	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7010	{
7011	NOREF(pMixedCtx);
7012
7013	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
7014	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
7015	return VINF_SUCCESS;
7016	}
7017
7018
7019	/**
7020	* Saves the entire guest state from the currently active VMCS into the
7021	* guest-CPU context.
7022	*
7023	* This essentially VMREADs all guest-data.
7024	*
7025	* @returns VBox status code.
7026	* @param pVCpu The cross context virtual CPU structure.
7027	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7028	* out-of-sync. Make sure to update the required fields
7029	* before using them.
7030	*/
7031	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7032	{
7033	Assert(pVCpu);
7034	Assert(pMixedCtx);
7035
7036	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7037	return VINF_SUCCESS;
7038
7039	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7040	again on the ring-3 callback path, there is no real need to. */
7041	if (VMMRZCallRing3IsEnabled(pVCpu))
7042	VMMR0LogFlushDisable(pVCpu);
7043	else
7044	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7045	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7046
7047	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7048	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7049
7050	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7051	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7052
7053	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7054	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7055
7056	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7057	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7058
7059	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7060	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7061
7062	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7063	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7064
7065	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7066	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7067
7068	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7069	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7070
7071	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7072	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7073
7074	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7075	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7076
7077	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7078	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7079	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7080
7081	if (VMMRZCallRing3IsEnabled(pVCpu))
7082	VMMR0LogFlushEnable(pVCpu);
7083
7084	return VINF_SUCCESS;
7085	}
7086
7087
7088	/**
7089	* Saves basic guest registers needed for IEM instruction execution.
7090	*
7091	* @returns VBox status code (OR-able).
7092	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7093	* @param pMixedCtx Pointer to the CPU context of the guest.
7094	* @param fMemory Whether the instruction being executed operates on
7095	* memory or not. Only CR0 is synced up if clear.
7096	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7097	*/
7098	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7099	{
7100	/*
7101	* We assume all general purpose registers other than RSP are available.
7102	*
7103	* - RIP is a must, as it will be incremented or otherwise changed.
7104	* - RFLAGS are always required to figure the CPL.
7105	* - RSP isn't always required, however it's a GPR, so frequently required.
7106	* - SS and CS are the only segment register needed if IEM doesn't do memory
7107	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7108	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7109	* be required for memory accesses.
7110	*
7111	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7112	*/
7113	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7114	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7115	if (fNeedRsp)
7116	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7117	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7118	if (!fMemory)
7119	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7120	else
7121	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7122	AssertRCReturn(rc, rc);
7123	return rc;
7124	}
7125
7126
7127	/**
7128	* Ensures that we've got a complete basic guest-context.
7129	*
7130	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7131	* is for the interpreter.
7132	*
7133	* @returns VBox status code.
7134	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7135	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7136	* needing to be synced in.
7137	* @thread EMT(pVCpu)
7138	*/
7139	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7140	{
7141	/* Note! Since this is only applicable to VT-x, the implementation is placed
7142	in the VT-x part of the sources instead of the generic stuff. */
7143	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7144	{
7145	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7146	/*
7147	* For now, imply that the caller might change everything too. Do this after
7148	* saving the guest state so as to not trigger assertions.
7149	*/
7150	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7151	return rc;
7152	}
7153	return VINF_SUCCESS;
7154	}
7155
7156
7157	/**
7158	* Check per-VM and per-VCPU force flag actions that require us to go back to
7159	* ring-3 for one reason or another.
7160	*
7161	* @returns Strict VBox status code (i.e. informational status codes too)
7162	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7163	* ring-3.
7164	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7165	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7166	* interrupts)
7167	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7168	* all EMTs to be in ring-3.
7169	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7170	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7171	* to the EM loop.
7172	*
7173	* @param pVM The cross context VM structure.
7174	* @param pVCpu The cross context virtual CPU structure.
7175	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7176	* out-of-sync. Make sure to update the required fields
7177	* before using them.
7178	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7179	*/
7180	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7181	{
7182	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7183
7184	/*
7185	* Anything pending? Should be more likely than not if we're doing a good job.
7186	*/
7187	if ( !fStepping
7188	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7189	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7190	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7191	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7192	return VINF_SUCCESS;
7193
7194	/* We need the control registers now, make sure the guest-CPU context is updated. */
7195	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7196	AssertRCReturn(rc3, rc3);
7197
7198	/* Pending HM CR3 sync. */
7199	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7200	{
7201	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7202	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7203	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7204	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7205	}
7206
7207	/* Pending HM PAE PDPEs. */
7208	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7209	{
7210	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7211	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7212	}
7213
7214	/* Pending PGM C3 sync. */
7215	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7216	{
7217	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7218	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7219	if (rcStrict2 != VINF_SUCCESS)
7220	{
7221	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7222	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7223	return rcStrict2;
7224	}
7225	}
7226
7227	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7228	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7229	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7230	{
7231	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7232	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7233	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7234	return rc2;
7235	}
7236
7237	/* Pending VM request packets, such as hardware interrupts. */
7238	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7239	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7240	{
7241	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7242	return VINF_EM_PENDING_REQUEST;
7243	}
7244
7245	/* Pending PGM pool flushes. */
7246	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7247	{
7248	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7249	return VINF_PGM_POOL_FLUSH_PENDING;
7250	}
7251
7252	/* Pending DMA requests. */
7253	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7254	{
7255	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7256	return VINF_EM_RAW_TO_R3;
7257	}
7258
7259	return VINF_SUCCESS;
7260	}
7261
7262
7263	/**
7264	* Converts any TRPM trap into a pending HM event. This is typically used when
7265	* entering from ring-3 (not longjmp returns).
7266	*
7267	* @param pVCpu The cross context virtual CPU structure.
7268	*/
7269	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7270	{
7271	Assert(TRPMHasTrap(pVCpu));
7272	Assert(!pVCpu->hm.s.Event.fPending);
7273
7274	uint8_t uVector;
7275	TRPMEVENT enmTrpmEvent;
7276	RTGCUINT uErrCode;
7277	RTGCUINTPTR GCPtrFaultAddress;
7278	uint8_t cbInstr;
7279
7280	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7281	AssertRC(rc);
7282
7283	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7284	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7285	if (enmTrpmEvent == TRPM_TRAP)
7286	{
7287	switch (uVector)
7288	{
7289	case X86_XCPT_NMI:
7290	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7291	break;
7292
7293	case X86_XCPT_BP:
7294	case X86_XCPT_OF:
7295	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7296	break;
7297
7298	case X86_XCPT_PF:
7299	case X86_XCPT_DF:
7300	case X86_XCPT_TS:
7301	case X86_XCPT_NP:
7302	case X86_XCPT_SS:
7303	case X86_XCPT_GP:
7304	case X86_XCPT_AC:
7305	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7306	RT_FALL_THRU();
7307	default:
7308	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7309	break;
7310	}
7311	}
7312	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7313	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7314	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7315	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7316	else
7317	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7318
7319	rc = TRPMResetTrap(pVCpu);
7320	AssertRC(rc);
7321	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7322	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7323
7324	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7325	}
7326
7327
7328	/**
7329	* Converts the pending HM event into a TRPM trap.
7330	*
7331	* @param pVCpu The cross context virtual CPU structure.
7332	*/
7333	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7334	{
7335	Assert(pVCpu->hm.s.Event.fPending);
7336
7337	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7338	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7339	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7340	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7341
7342	/* If a trap was already pending, we did something wrong! */
7343	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7344
7345	TRPMEVENT enmTrapType;
7346	switch (uVectorType)
7347	{
7348	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7349	enmTrapType = TRPM_HARDWARE_INT;
7350	break;
7351
7352	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7353	enmTrapType = TRPM_SOFTWARE_INT;
7354	break;
7355
7356	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7357	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7358	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7359	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7360	enmTrapType = TRPM_TRAP;
7361	break;
7362
7363	default:
7364	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7365	enmTrapType = TRPM_32BIT_HACK;
7366	break;
7367	}
7368
7369	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7370
7371	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7372	AssertRC(rc);
7373
7374	if (fErrorCodeValid)
7375	TRPMSetErrorCode(pVCpu, uErrorCode);
7376
7377	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7378	&& uVector == X86_XCPT_PF)
7379	{
7380	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7381	}
7382	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7383	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7384	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7385	{
7386	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7387	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7388	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7389	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7390	}
7391
7392	/* Clear any pending events from the VMCS. */
7393	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7394	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7395
7396	/* We're now done converting the pending event. */
7397	pVCpu->hm.s.Event.fPending = false;
7398	}
7399
7400
7401	/**
7402	* Does the necessary state syncing before returning to ring-3 for any reason
7403	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7404	*
7405	* @returns VBox status code.
7406	* @param pVCpu The cross context virtual CPU structure.
7407	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7408	* be out-of-sync. Make sure to update the required
7409	* fields before using them.
7410	* @param fSaveGuestState Whether to save the guest state or not.
7411	*
7412	* @remarks No-long-jmp zone!!!
7413	*/
7414	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7415	{
7416	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7417	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7418
7419	RTCPUID idCpu = RTMpCpuId();
7420	Log4Func(("HostCpuId=%u\n", idCpu));
7421
7422	/*
7423	* !!! IMPORTANT !!!
7424	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7425	*/
7426
7427	/* Save the guest state if necessary. */
7428	if ( fSaveGuestState
7429	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7430	{
7431	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7432	AssertRCReturn(rc, rc);
7433	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7434	}
7435
7436	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7437	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7438	{
7439	/* We shouldn't reload CR0 without saving it first. */
7440	if (!fSaveGuestState)
7441	{
7442	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7443	AssertRCReturn(rc, rc);
7444	}
7445	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7446	}
7447
7448	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7449	#ifdef VBOX_STRICT
7450	if (CPUMIsHyperDebugStateActive(pVCpu))
7451	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7452	#endif
7453	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7454	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7455	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7456	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7457
7458	#if HC_ARCH_BITS == 64
7459	/* Restore host-state bits that VT-x only restores partially. */
7460	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7461	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7462	{
7463	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7464	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7465	}
7466	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7467	#endif
7468
7469	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7470	if (pVCpu->hm.s.vmx.fLazyMsrs)
7471	{
7472	/* We shouldn't reload the guest MSRs without saving it first. */
7473	if (!fSaveGuestState)
7474	{
7475	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7476	AssertRCReturn(rc, rc);
7477	}
7478	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7479	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7480	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7481	}
7482
7483	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7484	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7485
7486	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7487	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7488	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7489	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7490	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7491	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7492	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7493	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7494
7495	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7496
7497	/** @todo This partially defeats the purpose of having preemption hooks.
7498	* The problem is, deregistering the hooks should be moved to a place that
7499	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7500	* context.
7501	*/
7502	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7503	{
7504	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7505	AssertRCReturn(rc, rc);
7506
7507	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7508	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7509	}
7510	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7511	NOREF(idCpu);
7512
7513	return VINF_SUCCESS;
7514	}
7515
7516
7517	/**
7518	* Leaves the VT-x session.
7519	*
7520	* @returns VBox status code.
7521	* @param pVCpu The cross context virtual CPU structure.
7522	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7523	* out-of-sync. Make sure to update the required fields
7524	* before using them.
7525	*
7526	* @remarks No-long-jmp zone!!!
7527	*/
7528	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7529	{
7530	HM_DISABLE_PREEMPT();
7531	HMVMX_ASSERT_CPU_SAFE();
7532	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7533	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7534
7535	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7536	and done this from the VMXR0ThreadCtxCallback(). */
7537	if (!pVCpu->hm.s.fLeaveDone)
7538	{
7539	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7540	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7541	pVCpu->hm.s.fLeaveDone = true;
7542	}
7543	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7544
7545	/*
7546	* !!! IMPORTANT !!!
7547	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7548	*/
7549
7550	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7551	/** @todo Deregistering here means we need to VMCLEAR always
7552	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7553	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7554	VMMR0ThreadCtxHookDisable(pVCpu);
7555
7556	/* Leave HM context. This takes care of local init (term). */
7557	int rc = HMR0LeaveCpu(pVCpu);
7558
7559	HM_RESTORE_PREEMPT();
7560	return rc;
7561	}
7562
7563
7564	/**
7565	* Does the necessary state syncing before doing a longjmp to ring-3.
7566	*
7567	* @returns VBox status code.
7568	* @param pVCpu The cross context virtual CPU structure.
7569	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7570	* out-of-sync. Make sure to update the required fields
7571	* before using them.
7572	*
7573	* @remarks No-long-jmp zone!!!
7574	*/
7575	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7576	{
7577	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7578	}
7579
7580
7581	/**
7582	* Take necessary actions before going back to ring-3.
7583	*
7584	* An action requires us to go back to ring-3. This function does the necessary
7585	* steps before we can safely return to ring-3. This is not the same as longjmps
7586	* to ring-3, this is voluntary and prepares the guest so it may continue
7587	* executing outside HM (recompiler/IEM).
7588	*
7589	* @returns VBox status code.
7590	* @param pVM The cross context VM structure.
7591	* @param pVCpu The cross context virtual CPU structure.
7592	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7593	* out-of-sync. Make sure to update the required fields
7594	* before using them.
7595	* @param rcExit The reason for exiting to ring-3. Can be
7596	* VINF_VMM_UNKNOWN_RING3_CALL.
7597	*/
7598	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7599	{
7600	Assert(pVM);
7601	Assert(pVCpu);
7602	Assert(pMixedCtx);
7603	HMVMX_ASSERT_PREEMPT_SAFE();
7604
7605	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7606	{
7607	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7608	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7609	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7610	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7611	}
7612
7613	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7614	VMMRZCallRing3Disable(pVCpu);
7615	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7616
7617	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7618	if (pVCpu->hm.s.Event.fPending)
7619	{
7620	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7621	Assert(!pVCpu->hm.s.Event.fPending);
7622	}
7623
7624	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7625	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7626
7627	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7628	and if we're injecting an event we should have a TRPM trap pending. */
7629	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7630	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7631	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7632	#endif
7633
7634	/* Save guest state and restore host state bits. */
7635	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7636	AssertRCReturn(rc, rc);
7637	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7638	/* Thread-context hooks are unregistered at this point!!! */
7639
7640	/* Sync recompiler state. */
7641	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7642	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7643	\| CPUM_CHANGED_LDTR
7644	\| CPUM_CHANGED_GDTR
7645	\| CPUM_CHANGED_IDTR
7646	\| CPUM_CHANGED_TR
7647	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7648	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7649	if ( pVM->hm.s.fNestedPaging
7650	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7651	{
7652	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7653	}
7654
7655	Assert(!pVCpu->hm.s.fClearTrapFlag);
7656
7657	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7658	if (rcExit != VINF_EM_RAW_INTERRUPT)
7659	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7660
7661	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7662
7663	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7664	VMMRZCallRing3RemoveNotification(pVCpu);
7665	VMMRZCallRing3Enable(pVCpu);
7666
7667	return rc;
7668	}
7669
7670
7671	/**
7672	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7673	* longjump to ring-3 and possibly get preempted.
7674	*
7675	* @returns VBox status code.
7676	* @param pVCpu The cross context virtual CPU structure.
7677	* @param enmOperation The operation causing the ring-3 longjump.
7678	* @param pvUser Opaque pointer to the guest-CPU context. The data
7679	* may be out-of-sync. Make sure to update the required
7680	* fields before using them.
7681	*/
7682	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7683	{
7684	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7685	{
7686	/*
7687	* !!! IMPORTANT !!!
7688	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7689	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7690	*/
7691	VMMRZCallRing3RemoveNotification(pVCpu);
7692	VMMRZCallRing3Disable(pVCpu);
7693	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7694	RTThreadPreemptDisable(&PreemptState);
7695
7696	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7697	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7698
7699	#if HC_ARCH_BITS == 64
7700	/* Restore host-state bits that VT-x only restores partially. */
7701	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7702	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7703	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7704	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7705	#endif
7706	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7707	if (pVCpu->hm.s.vmx.fLazyMsrs)
7708	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7709
7710	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7711	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7712	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7713	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7714	{
7715	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7716	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7717	}
7718
7719	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7720	VMMR0ThreadCtxHookDisable(pVCpu);
7721	HMR0LeaveCpu(pVCpu);
7722	RTThreadPreemptRestore(&PreemptState);
7723	return VINF_SUCCESS;
7724	}
7725
7726	Assert(pVCpu);
7727	Assert(pvUser);
7728	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7729	HMVMX_ASSERT_PREEMPT_SAFE();
7730
7731	VMMRZCallRing3Disable(pVCpu);
7732	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7733
7734	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7735	enmOperation));
7736
7737	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7738	AssertRCReturn(rc, rc);
7739
7740	VMMRZCallRing3Enable(pVCpu);
7741	return VINF_SUCCESS;
7742	}
7743
7744
7745	/**
7746	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7747	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7748	*
7749	* @param pVCpu The cross context virtual CPU structure.
7750	*/
7751	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7752	{
7753	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7754	{
7755	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7756	{
7757	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7758	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7759	AssertRC(rc);
7760	Log4(("Setup interrupt-window exiting\n"));
7761	}
7762	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7763	}
7764
7765
7766	/**
7767	* Clears the interrupt-window exiting control in the VMCS.
7768	*
7769	* @param pVCpu The cross context virtual CPU structure.
7770	*/
7771	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7772	{
7773	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7774	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7775	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7776	AssertRC(rc);
7777	Log4(("Cleared interrupt-window exiting\n"));
7778	}
7779
7780
7781	/**
7782	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7783	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7784	*
7785	* @param pVCpu The cross context virtual CPU structure.
7786	*/
7787	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7788	{
7789	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7790	{
7791	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7792	{
7793	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7794	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7795	AssertRC(rc);
7796	Log4(("Setup NMI-window exiting\n"));
7797	}
7798	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7799	}
7800
7801
7802	/**
7803	* Clears the NMI-window exiting control in the VMCS.
7804	*
7805	* @param pVCpu The cross context virtual CPU structure.
7806	*/
7807	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7808	{
7809	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7810	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7811	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7812	AssertRC(rc);
7813	Log4(("Cleared NMI-window exiting\n"));
7814	}
7815
7816
7817	/**
7818	* Evaluates the event to be delivered to the guest and sets it as the pending
7819	* event.
7820	*
7821	* @returns The VT-x guest-interruptibility state.
7822	* @param pVCpu The cross context virtual CPU structure.
7823	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7824	* out-of-sync. Make sure to update the required fields
7825	* before using them.
7826	*/
7827	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7828	{
7829	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7830	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7831	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7832	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7833	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7834
7835	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7836	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7837	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7838	Assert(!TRPMHasTrap(pVCpu));
7839
7840	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7841	APICUpdatePendingInterrupts(pVCpu);
7842
7843	/*
7844	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7845	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7846	*/
7847	/** @todo SMI. SMIs take priority over NMIs. */
7848	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7849	{
7850	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7851	if ( !pVCpu->hm.s.Event.fPending
7852	&& !fBlockNmi
7853	&& !fBlockSti
7854	&& !fBlockMovSS)
7855	{
7856	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7857	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7858	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7859
7860	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7861	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7862	}
7863	else
7864	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7865	}
7866	/*
7867	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7868	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7869	*/
7870	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7871	&& !pVCpu->hm.s.fSingleInstruction)
7872	{
7873	Assert(!DBGFIsStepping(pVCpu));
7874	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7875	AssertRC(rc);
7876	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7877	if ( !pVCpu->hm.s.Event.fPending
7878	&& !fBlockInt
7879	&& !fBlockSti
7880	&& !fBlockMovSS)
7881	{
7882	uint8_t u8Interrupt;
7883	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7884	if (RT_SUCCESS(rc))
7885	{
7886	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7887	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7888	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7889
7890	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7891	}
7892	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7893	{
7894	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7895	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7896	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7897
7898	/*
7899	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7900	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7901	* need to re-set this force-flag here.
7902	*/
7903	}
7904	else
7905	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7906	}
7907	else
7908	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7909	}
7910
7911	return uIntrState;
7912	}
7913
7914
7915	/**
7916	* Sets a pending-debug exception to be delivered to the guest if the guest is
7917	* single-stepping in the VMCS.
7918	*
7919	* @param pVCpu The cross context virtual CPU structure.
7920	*/
7921	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7922	{
7923	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7924	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7925	AssertRC(rc);
7926	}
7927
7928
7929	/**
7930	* Injects any pending events into the guest if the guest is in a state to
7931	* receive them.
7932	*
7933	* @returns Strict VBox status code (i.e. informational status codes too).
7934	* @param pVCpu The cross context virtual CPU structure.
7935	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7936	* out-of-sync. Make sure to update the required fields
7937	* before using them.
7938	* @param uIntrState The VT-x guest-interruptibility state.
7939	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7940	* return VINF_EM_DBG_STEPPED if the event was
7941	* dispatched directly.
7942	*/
7943	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7944	{
7945	HMVMX_ASSERT_PREEMPT_SAFE();
7946	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7947
7948	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7949	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7950
7951	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7952	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7953	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7954	Assert(!TRPMHasTrap(pVCpu));
7955
7956	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7957	if (pVCpu->hm.s.Event.fPending)
7958	{
7959	/*
7960	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7961	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7962	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7963	*
7964	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7965	*/
7966	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7967	#ifdef VBOX_STRICT
7968	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7969	{
7970	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7971	Assert(!fBlockInt);
7972	Assert(!fBlockSti);
7973	Assert(!fBlockMovSS);
7974	}
7975	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7976	{
7977	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7978	Assert(!fBlockSti);
7979	Assert(!fBlockMovSS);
7980	Assert(!fBlockNmi);
7981	}
7982	#endif
7983	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7984	(uint8_t)uIntType));
7985	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7986	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7987	fStepping, &uIntrState);
7988	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7989
7990	/* Update the interruptibility-state as it could have been changed by
7991	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7992	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7993	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7994
7995	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7996	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7997	else
7998	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7999	}
8000
8001	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
8002	if ( fBlockSti
8003	\|\| fBlockMovSS)
8004	{
8005	if (!pVCpu->hm.s.fSingleInstruction)
8006	{
8007	/*
8008	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
8009	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
8010	* See Intel spec. 27.3.4 "Saving Non-Register State".
8011	*/
8012	Assert(!DBGFIsStepping(pVCpu));
8013	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8014	AssertRCReturn(rc2, rc2);
8015	if (pMixedCtx->eflags.Bits.u1TF)
8016	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8017	}
8018	else if (pMixedCtx->eflags.Bits.u1TF)
8019	{
8020	/*
8021	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8022	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8023	*/
8024	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8025	uIntrState = 0;
8026	}
8027	}
8028
8029	/*
8030	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8031	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8032	*/
8033	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8034	AssertRC(rc2);
8035
8036	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8037	NOREF(fBlockMovSS); NOREF(fBlockSti);
8038	return rcStrict;
8039	}
8040
8041
8042	/**
8043	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8044	*
8045	* @param pVCpu The cross context virtual CPU structure.
8046	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8047	* out-of-sync. Make sure to update the required fields
8048	* before using them.
8049	*/
8050	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8051	{
8052	NOREF(pMixedCtx);
8053	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8054	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8055	}
8056
8057
8058	/**
8059	* Injects a double-fault (\#DF) exception into the VM.
8060	*
8061	* @returns Strict VBox status code (i.e. informational status codes too).
8062	* @param pVCpu The cross context virtual CPU structure.
8063	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8064	* out-of-sync. Make sure to update the required fields
8065	* before using them.
8066	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8067	* and should return VINF_EM_DBG_STEPPED if the event
8068	* is injected directly (register modified by us, not
8069	* by hardware on VM-entry).
8070	* @param puIntrState Pointer to the current guest interruptibility-state.
8071	* This interruptibility-state will be updated if
8072	* necessary. This cannot not be NULL.
8073	*/
8074	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8075	{
8076	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8077	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8078	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8079	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8080	fStepping, puIntrState);
8081	}
8082
8083
8084	/**
8085	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8086	*
8087	* @param pVCpu The cross context virtual CPU structure.
8088	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8089	* out-of-sync. Make sure to update the required fields
8090	* before using them.
8091	*/
8092	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8093	{
8094	NOREF(pMixedCtx);
8095	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8096	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8097	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8098	}
8099
8100
8101	/**
8102	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8103	*
8104	* @param pVCpu The cross context virtual CPU structure.
8105	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8106	* out-of-sync. Make sure to update the required fields
8107	* before using them.
8108	* @param cbInstr The value of RIP that is to be pushed on the guest
8109	* stack.
8110	*/
8111	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8112	{
8113	NOREF(pMixedCtx);
8114	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8115	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8116	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8117	}
8118
8119
8120	/**
8121	* Injects a general-protection (\#GP) fault into the VM.
8122	*
8123	* @returns Strict VBox status code (i.e. informational status codes too).
8124	* @param pVCpu The cross context virtual CPU structure.
8125	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8126	* out-of-sync. Make sure to update the required fields
8127	* before using them.
8128	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8129	* mode, i.e. in real-mode it's not valid).
8130	* @param u32ErrorCode The error code associated with the \#GP.
8131	* @param fStepping Whether we're running in
8132	* hmR0VmxRunGuestCodeStep() and should return
8133	* VINF_EM_DBG_STEPPED if the event is injected
8134	* directly (register modified by us, not by
8135	* hardware on VM-entry).
8136	* @param puIntrState Pointer to the current guest interruptibility-state.
8137	* This interruptibility-state will be updated if
8138	* necessary. This cannot not be NULL.
8139	*/
8140	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8141	bool fStepping, uint32_t *puIntrState)
8142	{
8143	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8144	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8145	if (fErrorCodeValid)
8146	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8147	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8148	fStepping, puIntrState);
8149	}
8150
8151
8152	#if 0 /* unused */
8153	/**
8154	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8155	* VM.
8156	*
8157	* @param pVCpu The cross context virtual CPU structure.
8158	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8159	* out-of-sync. Make sure to update the required fields
8160	* before using them.
8161	* @param u32ErrorCode The error code associated with the \#GP.
8162	*/
8163	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8164	{
8165	NOREF(pMixedCtx);
8166	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8167	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8168	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8169	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8170	}
8171	#endif /* unused */
8172
8173
8174	/**
8175	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8176	*
8177	* @param pVCpu The cross context virtual CPU structure.
8178	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8179	* out-of-sync. Make sure to update the required fields
8180	* before using them.
8181	* @param uVector The software interrupt vector number.
8182	* @param cbInstr The value of RIP that is to be pushed on the guest
8183	* stack.
8184	*/
8185	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8186	{
8187	NOREF(pMixedCtx);
8188	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8189	if ( uVector == X86_XCPT_BP
8190	\|\| uVector == X86_XCPT_OF)
8191	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8192	else
8193	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8194	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8195	}
8196
8197
8198	/**
8199	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8200	* stack.
8201	*
8202	* @returns Strict VBox status code (i.e. informational status codes too).
8203	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8204	* @param pVM The cross context VM structure.
8205	* @param pMixedCtx Pointer to the guest-CPU context.
8206	* @param uValue The value to push to the guest stack.
8207	*/
8208	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8209	{
8210	/*
8211	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8212	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8213	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8214	*/
8215	if (pMixedCtx->sp == 1)
8216	return VINF_EM_RESET;
8217	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8218	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8219	AssertRC(rc);
8220	return rc;
8221	}
8222
8223
8224	/**
8225	* Injects an event into the guest upon VM-entry by updating the relevant fields
8226	* in the VM-entry area in the VMCS.
8227	*
8228	* @returns Strict VBox status code (i.e. informational status codes too).
8229	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8230	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8231	*
8232	* @param pVCpu The cross context virtual CPU structure.
8233	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8234	* be out-of-sync. Make sure to update the required
8235	* fields before using them.
8236	* @param u64IntInfo The VM-entry interruption-information field.
8237	* @param cbInstr The VM-entry instruction length in bytes (for
8238	* software interrupts, exceptions and privileged
8239	* software exceptions).
8240	* @param u32ErrCode The VM-entry exception error code.
8241	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8242	* @param puIntrState Pointer to the current guest interruptibility-state.
8243	* This interruptibility-state will be updated if
8244	* necessary. This cannot not be NULL.
8245	* @param fStepping Whether we're running in
8246	* hmR0VmxRunGuestCodeStep() and should return
8247	* VINF_EM_DBG_STEPPED if the event is injected
8248	* directly (register modified by us, not by
8249	* hardware on VM-entry).
8250	*
8251	* @remarks Requires CR0!
8252	*/
8253	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8254	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8255	uint32_t *puIntrState)
8256	{
8257	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8258	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8259	Assert(puIntrState);
8260	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8261
8262	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8263	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8264
8265	#ifdef VBOX_STRICT
8266	/*
8267	* Validate the error-code-valid bit for hardware exceptions.
8268	* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8269	*/
8270	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8271	&& !CPUMIsGuestInRealModeEx(pMixedCtx))
8272	{
8273	switch (uVector)
8274	{
8275	case X86_XCPT_PF:
8276	case X86_XCPT_DF:
8277	case X86_XCPT_TS:
8278	case X86_XCPT_NP:
8279	case X86_XCPT_SS:
8280	case X86_XCPT_GP:
8281	case X86_XCPT_AC:
8282	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8283	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8284	RT_FALL_THRU();
8285	default:
8286	break;
8287	}
8288	}
8289	#endif
8290
8291	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8292	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8293	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8294
8295	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8296
8297	/* We require CR0 to check if the guest is in real-mode. */
8298	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8299	AssertRCReturn(rc, rc);
8300
8301	/*
8302	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8303	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8304	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8305	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8306	*/
8307	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8308	{
8309	PVM pVM = pVCpu->CTX_SUFF(pVM);
8310	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8311	{
8312	Assert(PDMVmmDevHeapIsEnabled(pVM));
8313	Assert(pVM->hm.s.vmx.pRealModeTSS);
8314
8315	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8316	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8317	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8318	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8319	AssertRCReturn(rc, rc);
8320	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8321
8322	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8323	size_t const cbIdtEntry = sizeof(X86IDTR16);
8324	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8325	{
8326	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8327	if (uVector == X86_XCPT_DF)
8328	return VINF_EM_RESET;
8329
8330	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8331	if (uVector == X86_XCPT_GP)
8332	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8333
8334	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8335	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8336	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8337	fStepping, puIntrState);
8338	}
8339
8340	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8341	uint16_t uGuestIp = pMixedCtx->ip;
8342	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8343	{
8344	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8345	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8346	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8347	}
8348	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8349	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8350
8351	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8352	X86IDTR16 IdtEntry;
8353	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8354	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8355	AssertRCReturn(rc, rc);
8356
8357	/* Construct the stack frame for the interrupt/exception handler. */
8358	VBOXSTRICTRC rcStrict;
8359	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8360	if (rcStrict == VINF_SUCCESS)
8361	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8362	if (rcStrict == VINF_SUCCESS)
8363	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8364
8365	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8366	if (rcStrict == VINF_SUCCESS)
8367	{
8368	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8369	pMixedCtx->rip = IdtEntry.offSel;
8370	pMixedCtx->cs.Sel = IdtEntry.uSel;
8371	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8372	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8373	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8374	&& uVector == X86_XCPT_PF)
8375	pMixedCtx->cr2 = GCPtrFaultAddress;
8376
8377	/* If any other guest-state bits are changed here, make sure to update
8378	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8379	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8380	\| HM_CHANGED_GUEST_RIP
8381	\| HM_CHANGED_GUEST_RFLAGS
8382	\| HM_CHANGED_GUEST_RSP);
8383
8384	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8385	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8386	{
8387	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8388	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8389	Log4(("Clearing inhibition due to STI.\n"));
8390	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8391	}
8392	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8393	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8394
8395	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8396	it, if we are returning to ring-3 before executing guest code. */
8397	pVCpu->hm.s.Event.fPending = false;
8398
8399	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8400	if (fStepping)
8401	rcStrict = VINF_EM_DBG_STEPPED;
8402	}
8403	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8404	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8405	return rcStrict;
8406	}
8407
8408	/*
8409	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8410	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8411	*/
8412	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8413	}
8414
8415	/* Validate. */
8416	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8417	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8418	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8419
8420	/* Inject. */
8421	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8422	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8423	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8424	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8425
8426	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8427	&& uVector == X86_XCPT_PF)
8428	pMixedCtx->cr2 = GCPtrFaultAddress;
8429
8430	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8431	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8432
8433	AssertRCReturn(rc, rc);
8434	return VINF_SUCCESS;
8435	}
8436
8437
8438	/**
8439	* Clears the interrupt-window exiting control in the VMCS and if necessary
8440	* clears the current event in the VMCS as well.
8441	*
8442	* @returns VBox status code.
8443	* @param pVCpu The cross context virtual CPU structure.
8444	*
8445	* @remarks Use this function only to clear events that have not yet been
8446	* delivered to the guest but are injected in the VMCS!
8447	* @remarks No-long-jump zone!!!
8448	*/
8449	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8450	{
8451	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8452
8453	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8454	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8455
8456	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8457	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8458	}
8459
8460
8461	/**
8462	* Enters the VT-x session.
8463	*
8464	* @returns VBox status code.
8465	* @param pVM The cross context VM structure.
8466	* @param pVCpu The cross context virtual CPU structure.
8467	* @param pCpu Pointer to the CPU info struct.
8468	*/
8469	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8470	{
8471	AssertPtr(pVM);
8472	AssertPtr(pVCpu);
8473	Assert(pVM->hm.s.vmx.fSupported);
8474	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8475	NOREF(pCpu); NOREF(pVM);
8476
8477	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8478	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8479
8480	#ifdef VBOX_STRICT
8481	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8482	RTCCUINTREG uHostCR4 = ASMGetCR4();
8483	if (!(uHostCR4 & X86_CR4_VMXE))
8484	{
8485	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8486	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8487	}
8488	#endif
8489
8490	/*
8491	* Load the VCPU's VMCS as the current (and active) one.
8492	*/
8493	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8494	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8495	if (RT_FAILURE(rc))
8496	return rc;
8497
8498	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8499	pVCpu->hm.s.fLeaveDone = false;
8500	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8501
8502	return VINF_SUCCESS;
8503	}
8504
8505
8506	/**
8507	* The thread-context callback (only on platforms which support it).
8508	*
8509	* @param enmEvent The thread-context event.
8510	* @param pVCpu The cross context virtual CPU structure.
8511	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8512	* @thread EMT(pVCpu)
8513	*/
8514	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8515	{
8516	NOREF(fGlobalInit);
8517
8518	switch (enmEvent)
8519	{
8520	case RTTHREADCTXEVENT_OUT:
8521	{
8522	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8523	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8524	VMCPU_ASSERT_EMT(pVCpu);
8525
8526	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8527
8528	/* No longjmps (logger flushes, locks) in this fragile context. */
8529	VMMRZCallRing3Disable(pVCpu);
8530	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8531
8532	/*
8533	* Restore host-state (FPU, debug etc.)
8534	*/
8535	if (!pVCpu->hm.s.fLeaveDone)
8536	{
8537	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8538	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8539	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8540	pVCpu->hm.s.fLeaveDone = true;
8541	}
8542
8543	/* Leave HM context, takes care of local init (term). */
8544	int rc = HMR0LeaveCpu(pVCpu);
8545	AssertRC(rc); NOREF(rc);
8546
8547	/* Restore longjmp state. */
8548	VMMRZCallRing3Enable(pVCpu);
8549	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8550	break;
8551	}
8552
8553	case RTTHREADCTXEVENT_IN:
8554	{
8555	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8556	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8557	VMCPU_ASSERT_EMT(pVCpu);
8558
8559	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8560	VMMRZCallRing3Disable(pVCpu);
8561	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8562
8563	/* Initialize the bare minimum state required for HM. This takes care of
8564	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8565	int rc = HMR0EnterCpu(pVCpu);
8566	AssertRC(rc);
8567	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8568
8569	/* Load the active VMCS as the current one. */
8570	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8571	{
8572	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8573	AssertRC(rc); NOREF(rc);
8574	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8575	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8576	}
8577	pVCpu->hm.s.fLeaveDone = false;
8578
8579	/* Restore longjmp state. */
8580	VMMRZCallRing3Enable(pVCpu);
8581	break;
8582	}
8583
8584	default:
8585	break;
8586	}
8587	}
8588
8589
8590	/**
8591	* Saves the host state in the VMCS host-state.
8592	* Sets up the VM-exit MSR-load area.
8593	*
8594	* The CPU state will be loaded from these fields on every successful VM-exit.
8595	*
8596	* @returns VBox status code.
8597	* @param pVM The cross context VM structure.
8598	* @param pVCpu The cross context virtual CPU structure.
8599	*
8600	* @remarks No-long-jump zone!!!
8601	*/
8602	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8603	{
8604	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8605
8606	int rc = VINF_SUCCESS;
8607	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8608	{
8609	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8610	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8611
8612	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8613	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8614
8615	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8616	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8617
8618	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8619	}
8620	return rc;
8621	}
8622
8623
8624	/**
8625	* Saves the host state in the VMCS host-state.
8626	*
8627	* @returns VBox status code.
8628	* @param pVM The cross context VM structure.
8629	* @param pVCpu The cross context virtual CPU structure.
8630	*
8631	* @remarks No-long-jump zone!!!
8632	*/
8633	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8634	{
8635	AssertPtr(pVM);
8636	AssertPtr(pVCpu);
8637
8638	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8639
8640	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8641	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8642	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8643	return hmR0VmxSaveHostState(pVM, pVCpu);
8644	}
8645
8646
8647	/**
8648	* Loads the guest state into the VMCS guest-state area.
8649	*
8650	* The will typically be done before VM-entry when the guest-CPU state and the
8651	* VMCS state may potentially be out of sync.
8652	*
8653	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8654	* VM-entry controls.
8655	* Sets up the appropriate VMX non-root function to execute guest code based on
8656	* the guest CPU mode.
8657	*
8658	* @returns VBox strict status code.
8659	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8660	* without unrestricted guest access and the VMMDev is not presently
8661	* mapped (e.g. EFI32).
8662	*
8663	* @param pVM The cross context VM structure.
8664	* @param pVCpu The cross context virtual CPU structure.
8665	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8666	* out-of-sync. Make sure to update the required fields
8667	* before using them.
8668	*
8669	* @remarks No-long-jump zone!!!
8670	*/
8671	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8672	{
8673	AssertPtr(pVM);
8674	AssertPtr(pVCpu);
8675	AssertPtr(pMixedCtx);
8676	HMVMX_ASSERT_PREEMPT_SAFE();
8677
8678	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8679
8680	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8681
8682	/* Determine real-on-v86 mode. */
8683	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8684	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8685	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8686	{
8687	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8688	}
8689
8690	/*
8691	* Load the guest-state into the VMCS.
8692	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8693	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8694	*/
8695	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8696	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8697
8698	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8699	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8700	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8701
8702	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8703	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8704	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8705
8706	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8707	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8708
8709	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8710	if (rcStrict == VINF_SUCCESS)
8711	{ /* likely */ }
8712	else
8713	{
8714	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8715	return rcStrict;
8716	}
8717
8718	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8719	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8720	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8721
8722	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8723	determine we don't have to swap EFER after all. */
8724	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8725	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8726
8727	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8728	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8729
8730	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8731	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8732
8733	/*
8734	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8735	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8736	*/
8737	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8738	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8739
8740	/* Clear any unused and reserved bits. */
8741	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8742
8743	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8744	return rc;
8745	}
8746
8747
8748	/**
8749	* Loads the state shared between the host and guest into the VMCS.
8750	*
8751	* @param pVM The cross context VM structure.
8752	* @param pVCpu The cross context virtual CPU structure.
8753	* @param pCtx Pointer to the guest-CPU context.
8754	*
8755	* @remarks No-long-jump zone!!!
8756	*/
8757	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8758	{
8759	NOREF(pVM);
8760
8761	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8762	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8763
8764	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8765	{
8766	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8767	AssertRC(rc);
8768	}
8769
8770	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8771	{
8772	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8773	AssertRC(rc);
8774
8775	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8776	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8777	{
8778	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8779	AssertRC(rc);
8780	}
8781	}
8782
8783	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8784	{
8785	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8786	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8787	}
8788
8789	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8790	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8791	{
8792	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8793	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8794	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8795	AssertRC(rc);
8796	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8797	}
8798
8799	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8800	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8801	}
8802
8803
8804	/**
8805	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8806	*
8807	* @returns Strict VBox status code (i.e. informational status codes too).
8808	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8809	* without unrestricted guest access and the VMMDev is not presently
8810	* mapped (e.g. EFI32).
8811	*
8812	* @param pVM The cross context VM structure.
8813	* @param pVCpu The cross context virtual CPU structure.
8814	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8815	* out-of-sync. Make sure to update the required fields
8816	* before using them.
8817	*
8818	* @remarks No-long-jump zone!!!
8819	*/
8820	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8821	{
8822	HMVMX_ASSERT_PREEMPT_SAFE();
8823	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8824	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8825
8826	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8827	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8828	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8829	#endif
8830
8831	/*
8832	* RIP is what changes the most often and hence if it's the only bit needing to be
8833	* updated, we shall handle it early for performance reasons.
8834	*/
8835	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8836	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8837	{
8838	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8839	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8840	{ /* likely */}
8841	else
8842	{
8843	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8844	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8845	}
8846	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8847	}
8848	else if (HMCPU_CF_VALUE(pVCpu))
8849	{
8850	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8851	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8852	{ /* likely */}
8853	else
8854	{
8855	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8856	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8857	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8858	return rcStrict;
8859	}
8860	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8861	}
8862
8863	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8864	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8865	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8866	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8867	return rcStrict;
8868	}
8869
8870
8871	/**
8872	* Does the preparations before executing guest code in VT-x.
8873	*
8874	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8875	* recompiler/IEM. We must be cautious what we do here regarding committing
8876	* guest-state information into the VMCS assuming we assuredly execute the
8877	* guest in VT-x mode.
8878	*
8879	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8880	* the common-state (TRPM/forceflags), we must undo those changes so that the
8881	* recompiler/IEM can (and should) use them when it resumes guest execution.
8882	* Otherwise such operations must be done when we can no longer exit to ring-3.
8883	*
8884	* @returns Strict VBox status code (i.e. informational status codes too).
8885	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8886	* have been disabled.
8887	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8888	* double-fault into the guest.
8889	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8890	* dispatched directly.
8891	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8892	*
8893	* @param pVM The cross context VM structure.
8894	* @param pVCpu The cross context virtual CPU structure.
8895	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8896	* out-of-sync. Make sure to update the required fields
8897	* before using them.
8898	* @param pVmxTransient Pointer to the VMX transient structure.
8899	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8900	* us ignore some of the reasons for returning to
8901	* ring-3, and return VINF_EM_DBG_STEPPED if event
8902	* dispatching took place.
8903	*/
8904	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8905	{
8906	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8907
8908	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8909	PGMRZDynMapFlushAutoSet(pVCpu);
8910	#endif
8911
8912	/* Check force flag actions that might require us to go back to ring-3. */
8913	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8914	if (rcStrict == VINF_SUCCESS)
8915	{ /* FFs doesn't get set all the time. */ }
8916	else
8917	return rcStrict;
8918
8919	#ifndef IEM_VERIFICATION_MODE_FULL
8920	/*
8921	* Setup the virtualized-APIC accesses.
8922	*
8923	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8924	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8925	*
8926	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
8927	*/
8928	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8929	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8930	&& PDMHasApic(pVM))
8931	{
8932	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8933	Assert(u64MsrApicBase);
8934	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8935
8936	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8937
8938	/* Unalias any existing mapping. */
8939	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8940	AssertRCReturn(rc, rc);
8941
8942	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8943	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8944	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8945	AssertRCReturn(rc, rc);
8946
8947	/* Update the per-VCPU cache of the APIC base MSR. */
8948	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8949	}
8950	#endif /* !IEM_VERIFICATION_MODE_FULL */
8951
8952	if (TRPMHasTrap(pVCpu))
8953	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8954	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8955
8956	/*
8957	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8958	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8959	*/
8960	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8961	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8962	{ /* likely */ }
8963	else
8964	{
8965	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8966	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8967	return rcStrict;
8968	}
8969
8970	/*
8971	* No longjmps to ring-3 from this point on!!!
8972	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8973	* This also disables flushing of the R0-logger instance (if any).
8974	*/
8975	VMMRZCallRing3Disable(pVCpu);
8976
8977	/*
8978	* Load the guest state bits.
8979	*
8980	* We cannot perform longjmps while loading the guest state because we do not preserve the
8981	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8982	* CPU migration.
8983	*
8984	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8985	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8986	* Hence, loading of the guest state needs to be done -after- injection of events.
8987	*/
8988	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8989	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8990	{ /* likely */ }
8991	else
8992	{
8993	VMMRZCallRing3Enable(pVCpu);
8994	return rcStrict;
8995	}
8996
8997	/*
8998	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8999	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
9000	*
9001	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
9002	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
9003	*
9004	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
9005	* executing guest code.
9006	*/
9007	pVmxTransient->fEFlags = ASMIntDisableFlags();
9008
9009	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
9010	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
9011	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
9012	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
9013	{
9014	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
9015	{
9016	pVCpu->hm.s.Event.fPending = false;
9017
9018	/*
9019	* We've injected any pending events. This is really the point of no return (to ring-3).
9020	*
9021	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
9022	* returns from this function, so don't enable them here.
9023	*/
9024	return VINF_SUCCESS;
9025	}
9026
9027	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9028	rcStrict = VINF_EM_RAW_INTERRUPT;
9029	}
9030	else
9031	{
9032	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9033	rcStrict = VINF_EM_RAW_TO_R3;
9034	}
9035
9036	ASMSetFlags(pVmxTransient->fEFlags);
9037	VMMRZCallRing3Enable(pVCpu);
9038
9039	return rcStrict;
9040	}
9041
9042
9043	/**
9044	* Prepares to run guest code in VT-x and we've committed to doing so. This
9045	* means there is no backing out to ring-3 or anywhere else at this
9046	* point.
9047	*
9048	* @param pVM The cross context VM structure.
9049	* @param pVCpu The cross context virtual CPU structure.
9050	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9051	* out-of-sync. Make sure to update the required fields
9052	* before using them.
9053	* @param pVmxTransient Pointer to the VMX transient structure.
9054	*
9055	* @remarks Called with preemption disabled.
9056	* @remarks No-long-jump zone!!!
9057	*/
9058	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9059	{
9060	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9061	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9062	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9063
9064	/*
9065	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9066	*/
9067	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9068	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9069
9070	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9071	if (!CPUMIsGuestFPUStateActive(pVCpu))
9072	{
9073	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9074	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9075	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9076	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9077	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9078	}
9079	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9080	#endif
9081
9082	if ( pVCpu->hm.s.fPreloadGuestFpu
9083	&& !CPUMIsGuestFPUStateActive(pVCpu))
9084	{
9085	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9086	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9087	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9088	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9089	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9090	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9091	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9092	}
9093
9094	/*
9095	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9096	*/
9097	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9098	&& pVCpu->hm.s.vmx.cMsrs > 0)
9099	{
9100	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9101	}
9102
9103	/*
9104	* Load the host state bits as we may've been preempted (only happens when
9105	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9106	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9107	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9108	* See @bugref{8432}.
9109	*/
9110	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9111	{
9112	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9113	AssertRC(rc);
9114	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9115	}
9116	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9117
9118	/*
9119	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9120	*/
9121	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9122	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9123	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9124
9125	/* Store status of the shared guest-host state at the time of VM-entry. */
9126	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9127	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9128	{
9129	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9130	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9131	}
9132	else
9133	#endif
9134	{
9135	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9136	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9137	}
9138	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9139
9140	/*
9141	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9142	*/
9143	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9144	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9145
9146	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9147	RTCPUID idCurrentCpu = pCpu->idCpu;
9148	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9149	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9150	{
9151	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9152	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9153	}
9154
9155	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9156	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9157	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9158	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9159
9160	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9161
9162	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9163	to start executing. */
9164
9165	/*
9166	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9167	*/
9168	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9169	{
9170	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9171	{
9172	bool fMsrUpdated;
9173	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9174	AssertRC(rc2);
9175	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9176
9177	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9178	&fMsrUpdated);
9179	AssertRC(rc2);
9180	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9181
9182	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9183	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9184	}
9185	else
9186	{
9187	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9188	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9189	}
9190	}
9191
9192	if (pVM->cpum.ro.GuestFeatures.fIbrs)
9193	{
9194	bool fMsrUpdated;
9195	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9196	AssertRC(rc2);
9197	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9198
9199	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMR0GetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
9200	&fMsrUpdated);
9201	AssertRC(rc2);
9202	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9203	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9204	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9205	}
9206
9207	#ifdef VBOX_STRICT
9208	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9209	hmR0VmxCheckHostEferMsr(pVCpu);
9210	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9211	#endif
9212	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9213	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9214	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9215	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9216	#endif
9217	}
9218
9219
9220	/**
9221	* Performs some essential restoration of state after running guest code in
9222	* VT-x.
9223	*
9224	* @param pVM The cross context VM structure.
9225	* @param pVCpu The cross context virtual CPU structure.
9226	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9227	* out-of-sync. Make sure to update the required fields
9228	* before using them.
9229	* @param pVmxTransient Pointer to the VMX transient structure.
9230	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9231	*
9232	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9233	*
9234	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9235	* unconditionally when it is safe to do so.
9236	*/
9237	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9238	{
9239	NOREF(pVM);
9240
9241	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9242
9243	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9244	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9245	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9246	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9247	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9248	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9249
9250	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9251	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9252
9253	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9254	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9255	Assert(!ASMIntAreEnabled());
9256	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9257
9258	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9259	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9260	{
9261	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9262	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9263	}
9264	#endif
9265
9266	#if HC_ARCH_BITS == 64
9267	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9268	#endif
9269	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9270	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9271	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9272	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9273	#else
9274	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9275	#endif
9276	#ifdef VBOX_STRICT
9277	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9278	#endif
9279	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9280
9281	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9282	uint32_t uExitReason;
9283	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9284	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9285	AssertRC(rc);
9286	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9287	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9288
9289	if (rcVMRun == VINF_SUCCESS)
9290	{
9291	/*
9292	* Update the VM-exit history array here even if the VM-entry failed due to:
9293	* - Invalid guest state.
9294	* - MSR loading.
9295	* - Machine-check event.
9296	*
9297	* In any of the above cases we will still have a "valid" VM-exit reason
9298	* despite @a fVMEntryFailed being false.
9299	*
9300	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9301	*/
9302	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9303
9304	if (!pVmxTransient->fVMEntryFailed)
9305	{
9306	/** @todo We can optimize this by only syncing with our force-flags when
9307	* really needed and keeping the VMCS state as it is for most
9308	* VM-exits. */
9309	/* Update the guest interruptibility-state from the VMCS. */
9310	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9311
9312	/*
9313	* Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9314	* as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9315	* on the longjmp path to ring-3 while saving the (rest of) the guest state,
9316	* see @bugref{6208#c63}.
9317	*/
9318	VMMRZCallRing3Enable(pVCpu);
9319
9320	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9321	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9322	AssertRC(rc);
9323	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9324	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9325	AssertRC(rc);
9326	#endif
9327
9328	/*
9329	* Sync the TPR shadow with our APIC state.
9330	*/
9331	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9332	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9333	{
9334	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9335	AssertRC(rc);
9336	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9337	}
9338
9339	return;
9340	}
9341	}
9342	else
9343	{
9344	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9345	pVmxTransient->fVMEntryFailed));
9346	}
9347
9348	VMMRZCallRing3Enable(pVCpu);
9349	}
9350
9351
9352	/**
9353	* Runs the guest code using VT-x the normal way.
9354	*
9355	* @returns VBox status code.
9356	* @param pVM The cross context VM structure.
9357	* @param pVCpu The cross context virtual CPU structure.
9358	* @param pCtx Pointer to the guest-CPU context.
9359	*
9360	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9361	*/
9362	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9363	{
9364	VMXTRANSIENT VmxTransient;
9365	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9366	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9367	uint32_t cLoops = 0;
9368
9369	for (;; cLoops++)
9370	{
9371	Assert(!HMR0SuspendPending());
9372	HMVMX_ASSERT_CPU_SAFE();
9373
9374	/* Preparatory work for running guest code, this may force us to return
9375	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9376	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9377	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9378	if (rcStrict != VINF_SUCCESS)
9379	break;
9380
9381	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9382	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9383	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9384
9385	/* Restore any residual host-state and save any bits shared between host
9386	and guest into the guest-CPU state. Re-enables interrupts! */
9387	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9388
9389	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9390	if (RT_SUCCESS(rcRun))
9391	{ /* very likely */ }
9392	else
9393	{
9394	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9395	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9396	return rcRun;
9397	}
9398
9399	/* Profile the VM-exit. */
9400	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9401	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9402	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9403	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9404	HMVMX_START_EXIT_DISPATCH_PROF();
9405
9406	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9407
9408	/* Handle the VM-exit. */
9409	#ifdef HMVMX_USE_FUNCTION_TABLE
9410	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9411	#else
9412	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9413	#endif
9414	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9415	if (rcStrict == VINF_SUCCESS)
9416	{
9417	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9418	continue; /* likely */
9419	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9420	rcStrict = VINF_EM_RAW_INTERRUPT;
9421	}
9422	break;
9423	}
9424
9425	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9426	return rcStrict;
9427	}
9428
9429
9430
9431	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9432	* probes.
9433	*
9434	* The following few functions and associated structure contains the bloat
9435	* necessary for providing detailed debug events and dtrace probes as well as
9436	* reliable host side single stepping. This works on the principle of
9437	* "subclassing" the normal execution loop and workers. We replace the loop
9438	* method completely and override selected helpers to add necessary adjustments
9439	* to their core operation.
9440	*
9441	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9442	* any performance for debug and analysis features.
9443	*
9444	* @{
9445	*/
9446
9447	/**
9448	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9449	* the debug run loop.
9450	*/
9451	typedef struct VMXRUNDBGSTATE
9452	{
9453	/** The RIP we started executing at. This is for detecting that we stepped. */
9454	uint64_t uRipStart;
9455	/** The CS we started executing with. */
9456	uint16_t uCsStart;
9457
9458	/** Whether we've actually modified the 1st execution control field. */
9459	bool fModifiedProcCtls : 1;
9460	/** Whether we've actually modified the 2nd execution control field. */
9461	bool fModifiedProcCtls2 : 1;
9462	/** Whether we've actually modified the exception bitmap. */
9463	bool fModifiedXcptBitmap : 1;
9464
9465	/** We desire the modified the CR0 mask to be cleared. */
9466	bool fClearCr0Mask : 1;
9467	/** We desire the modified the CR4 mask to be cleared. */
9468	bool fClearCr4Mask : 1;
9469	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9470	uint32_t fCpe1Extra;
9471	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9472	uint32_t fCpe1Unwanted;
9473	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9474	uint32_t fCpe2Extra;
9475	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9476	uint32_t bmXcptExtra;
9477	/** The sequence number of the Dtrace provider settings the state was
9478	* configured against. */
9479	uint32_t uDtraceSettingsSeqNo;
9480	/** VM-exits to check (one bit per VM-exit). */
9481	uint32_t bmExitsToCheck[3];
9482
9483	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9484	uint32_t fProcCtlsInitial;
9485	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9486	uint32_t fProcCtls2Initial;
9487	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9488	uint32_t bmXcptInitial;
9489	} VMXRUNDBGSTATE;
9490	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9491	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9492
9493
9494	/**
9495	* Initializes the VMXRUNDBGSTATE structure.
9496	*
9497	* @param pVCpu The cross context virtual CPU structure of the
9498	* calling EMT.
9499	* @param pCtx The CPU register context to go with @a pVCpu.
9500	* @param pDbgState The structure to initialize.
9501	*/
9502	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9503	{
9504	pDbgState->uRipStart = pCtx->rip;
9505	pDbgState->uCsStart = pCtx->cs.Sel;
9506
9507	pDbgState->fModifiedProcCtls = false;
9508	pDbgState->fModifiedProcCtls2 = false;
9509	pDbgState->fModifiedXcptBitmap = false;
9510	pDbgState->fClearCr0Mask = false;
9511	pDbgState->fClearCr4Mask = false;
9512	pDbgState->fCpe1Extra = 0;
9513	pDbgState->fCpe1Unwanted = 0;
9514	pDbgState->fCpe2Extra = 0;
9515	pDbgState->bmXcptExtra = 0;
9516	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9517	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9518	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9519	}
9520
9521
9522	/**
9523	* Updates the VMSC fields with changes requested by @a pDbgState.
9524	*
9525	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9526	* immediately before executing guest code, i.e. when interrupts are disabled.
9527	* We don't check status codes here as we cannot easily assert or return in the
9528	* latter case.
9529	*
9530	* @param pVCpu The cross context virtual CPU structure.
9531	* @param pDbgState The debug state.
9532	*/
9533	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9534	{
9535	/*
9536	* Ensure desired flags in VMCS control fields are set.
9537	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9538	*
9539	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9540	* there should be no stale data in pCtx at this point.
9541	*/
9542	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9543	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9544	{
9545	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9546	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9547	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9548	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9549	pDbgState->fModifiedProcCtls = true;
9550	}
9551
9552	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9553	{
9554	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9555	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9556	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9557	pDbgState->fModifiedProcCtls2 = true;
9558	}
9559
9560	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9561	{
9562	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9563	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9564	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9565	pDbgState->fModifiedXcptBitmap = true;
9566	}
9567
9568	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9569	{
9570	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9571	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9572	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9573	}
9574
9575	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9576	{
9577	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9578	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9579	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9580	}
9581	}
9582
9583
9584	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9585	{
9586	/*
9587	* Restore VM-exit control settings as we may not reenter this function the
9588	* next time around.
9589	*/
9590	/* We reload the initial value, trigger what we can of recalculations the
9591	next time around. From the looks of things, that's all that's required atm. */
9592	if (pDbgState->fModifiedProcCtls)
9593	{
9594	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9595	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9596	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9597	AssertRCReturn(rc2, rc2);
9598	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9599	}
9600
9601	/* We're currently the only ones messing with this one, so just restore the
9602	cached value and reload the field. */
9603	if ( pDbgState->fModifiedProcCtls2
9604	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9605	{
9606	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9607	AssertRCReturn(rc2, rc2);
9608	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9609	}
9610
9611	/* If we've modified the exception bitmap, we restore it and trigger
9612	reloading and partial recalculation the next time around. */
9613	if (pDbgState->fModifiedXcptBitmap)
9614	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9615
9616	return rcStrict;
9617	}
9618
9619
9620	/**
9621	* Configures VM-exit controls for current DBGF and DTrace settings.
9622	*
9623	* This updates @a pDbgState and the VMCS execution control fields to reflect
9624	* the necessary VM-exits demanded by DBGF and DTrace.
9625	*
9626	* @param pVM The cross context VM structure.
9627	* @param pVCpu The cross context virtual CPU structure.
9628	* @param pCtx Pointer to the guest-CPU context.
9629	* @param pDbgState The debug state.
9630	* @param pVmxTransient Pointer to the VMX transient structure. May update
9631	* fUpdateTscOffsettingAndPreemptTimer.
9632	*/
9633	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9634	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9635	{
9636	/*
9637	* Take down the dtrace serial number so we can spot changes.
9638	*/
9639	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9640	ASMCompilerBarrier();
9641
9642	/*
9643	* We'll rebuild most of the middle block of data members (holding the
9644	* current settings) as we go along here, so start by clearing it all.
9645	*/
9646	pDbgState->bmXcptExtra = 0;
9647	pDbgState->fCpe1Extra = 0;
9648	pDbgState->fCpe1Unwanted = 0;
9649	pDbgState->fCpe2Extra = 0;
9650	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9651	pDbgState->bmExitsToCheck[i] = 0;
9652
9653	/*
9654	* Software interrupts (INT XXh) - no idea how to trigger these...
9655	*/
9656	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9657	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9658	{
9659	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9660	}
9661
9662	/*
9663	* INT3 breakpoints - triggered by #BP exceptions.
9664	*/
9665	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9666	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9667
9668	/*
9669	* Exception bitmap and XCPT events+probes.
9670	*/
9671	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9672	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9673	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9674
9675	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9676	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9677	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9678	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9679	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9680	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9681	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9682	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9683	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9684	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9685	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9686	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9687	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9688	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9689	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9690	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9691	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9692	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9693
9694	if (pDbgState->bmXcptExtra)
9695	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9696
9697	/*
9698	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9699	*
9700	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9701	* So, when adding/changing/removing please don't forget to update it.
9702	*
9703	* Some of the macros are picking up local variables to save horizontal space,
9704	* (being able to see it in a table is the lesser evil here).
9705	*/
9706	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9707	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9708	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9709	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9710	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9711	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9712	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9713	} else do { } while (0)
9714	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9715	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9716	{ \
9717	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9718	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9719	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9720	} else do { } while (0)
9721	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9722	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9723	{ \
9724	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9725	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9726	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9727	} else do { } while (0)
9728	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9729	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9730	{ \
9731	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9732	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9733	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9734	} else do { } while (0)
9735
9736	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9737	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9738	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9739	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9740	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9741
9742	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9743	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9744	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9745	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9746	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9747	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9748	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9749	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9750	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9751	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9752	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9753	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9754	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9755	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9756	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9757	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9758	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9759	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9760	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9761	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9762	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9763	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9764	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9765	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9766	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9767	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9768	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9769	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9770	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9771	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9772	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9773	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9774	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9775	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9776	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9777	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9778
9779	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9780	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9781	{
9782	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9783	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9784	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9785	AssertRC(rc2);
9786
9787	#if 0 /** @todo fix me */
9788	pDbgState->fClearCr0Mask = true;
9789	pDbgState->fClearCr4Mask = true;
9790	#endif
9791	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9792	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9793	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9794	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9795	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9796	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9797	require clearing here and in the loop if we start using it. */
9798	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9799	}
9800	else
9801	{
9802	if (pDbgState->fClearCr0Mask)
9803	{
9804	pDbgState->fClearCr0Mask = false;
9805	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9806	}
9807	if (pDbgState->fClearCr4Mask)
9808	{
9809	pDbgState->fClearCr4Mask = false;
9810	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9811	}
9812	}
9813	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9814	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9815
9816	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9817	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9818	{
9819	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9820	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9821	}
9822	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9823	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9824
9825	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9826	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9827	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9828	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9829	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9830	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9831	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9832	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9833	#if 0 /** @todo too slow, fix handler. */
9834	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9835	#endif
9836	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9837
9838	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9839	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9840	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9841	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9842	{
9843	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9844	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9845	}
9846	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9847	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9848	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9849	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9850
9851	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9852	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9853	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9854	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9855	{
9856	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9857	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9858	}
9859	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9860	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9861	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9862	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9863
9864	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9865	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9866	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9867	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9868	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9869	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9870	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9871	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9872	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9873	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9874	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9875	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9876	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9877	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9878	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9879	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9880	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9881	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9882	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9883	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9884	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9885	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9886
9887	#undef IS_EITHER_ENABLED
9888	#undef SET_ONLY_XBM_IF_EITHER_EN
9889	#undef SET_CPE1_XBM_IF_EITHER_EN
9890	#undef SET_CPEU_XBM_IF_EITHER_EN
9891	#undef SET_CPE2_XBM_IF_EITHER_EN
9892
9893	/*
9894	* Sanitize the control stuff.
9895	*/
9896	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9897	if (pDbgState->fCpe2Extra)
9898	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9899	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9900	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9901	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9902	{
9903	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9904	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9905	}
9906
9907	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9908	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9909	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9910	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9911	}
9912
9913
9914	/**
9915	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9916	* appropriate.
9917	*
9918	* The caller has checked the VM-exit against the
9919	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9920	* already, so we don't have to do that either.
9921	*
9922	* @returns Strict VBox status code (i.e. informational status codes too).
9923	* @param pVM The cross context VM structure.
9924	* @param pVCpu The cross context virtual CPU structure.
9925	* @param pMixedCtx Pointer to the guest-CPU context.
9926	* @param pVmxTransient Pointer to the VMX-transient structure.
9927	* @param uExitReason The VM-exit reason.
9928	*
9929	* @remarks The name of this function is displayed by dtrace, so keep it short
9930	* and to the point. No longer than 33 chars long, please.
9931	*/
9932	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9933	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9934	{
9935	/*
9936	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9937	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9938	*
9939	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9940	* does. Must add/change/remove both places. Same ordering, please.
9941	*
9942	* Added/removed events must also be reflected in the next section
9943	* where we dispatch dtrace events.
9944	*/
9945	bool fDtrace1 = false;
9946	bool fDtrace2 = false;
9947	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9948	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9949	uint32_t uEventArg = 0;
9950	#define SET_EXIT(a_EventSubName) \
9951	do { \
9952	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9953	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9954	} while (0)
9955	#define SET_BOTH(a_EventSubName) \
9956	do { \
9957	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9958	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9959	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9960	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9961	} while (0)
9962	switch (uExitReason)
9963	{
9964	case VMX_EXIT_MTF:
9965	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9966
9967	case VMX_EXIT_XCPT_OR_NMI:
9968	{
9969	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9970	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9971	{
9972	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9973	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9974	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9975	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9976	{
9977	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9978	{
9979	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9980	uEventArg = pVmxTransient->uExitIntErrorCode;
9981	}
9982	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9983	switch (enmEvent1)
9984	{
9985	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9986	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9987	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9988	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9989	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9990	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9991	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9992	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9993	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9994	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9995	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9996	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9997	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9998	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9999	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
10000	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
10001	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
10002	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
10003	default: break;
10004	}
10005	}
10006	else
10007	AssertFailed();
10008	break;
10009
10010	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
10011	uEventArg = idxVector;
10012	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
10013	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
10014	break;
10015	}
10016	break;
10017	}
10018
10019	case VMX_EXIT_TRIPLE_FAULT:
10020	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
10021	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
10022	break;
10023	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
10024	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
10025	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
10026	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
10027	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
10028
10029	/* Instruction specific VM-exits: */
10030	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10031	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10032	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10033	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10034	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10035	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10036	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10037	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10038	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10039	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10040	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10041	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10042	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10043	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10044	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10045	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10046	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10047	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10048	case VMX_EXIT_MOV_CRX:
10049	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10050	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10051	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10052	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10053	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10054	SET_BOTH(CRX_READ);
10055	else
10056	SET_BOTH(CRX_WRITE);
10057	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10058	break;
10059	case VMX_EXIT_MOV_DRX:
10060	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10061	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10062	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10063	SET_BOTH(DRX_READ);
10064	else
10065	SET_BOTH(DRX_WRITE);
10066	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10067	break;
10068	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10069	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10070	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10071	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10072	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10073	case VMX_EXIT_XDTR_ACCESS:
10074	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10075	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10076	{
10077	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10078	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10079	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10080	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10081	}
10082	break;
10083
10084	case VMX_EXIT_TR_ACCESS:
10085	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10086	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10087	{
10088	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10089	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10090	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10091	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10092	}
10093	break;
10094
10095	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10096	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10097	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10098	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10099	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10100	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10101	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10102	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10103	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10104	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10105	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10106
10107	/* Events that aren't relevant at this point. */
10108	case VMX_EXIT_EXT_INT:
10109	case VMX_EXIT_INT_WINDOW:
10110	case VMX_EXIT_NMI_WINDOW:
10111	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10112	case VMX_EXIT_PREEMPT_TIMER:
10113	case VMX_EXIT_IO_INSTR:
10114	break;
10115
10116	/* Errors and unexpected events. */
10117	case VMX_EXIT_INIT_SIGNAL:
10118	case VMX_EXIT_SIPI:
10119	case VMX_EXIT_IO_SMI:
10120	case VMX_EXIT_SMI:
10121	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10122	case VMX_EXIT_ERR_MSR_LOAD:
10123	case VMX_EXIT_ERR_MACHINE_CHECK:
10124	break;
10125
10126	default:
10127	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10128	break;
10129	}
10130	#undef SET_BOTH
10131	#undef SET_EXIT
10132
10133	/*
10134	* Dtrace tracepoints go first. We do them here at once so we don't
10135	* have to copy the guest state saving and stuff a few dozen times.
10136	* Down side is that we've got to repeat the switch, though this time
10137	* we use enmEvent since the probes are a subset of what DBGF does.
10138	*/
10139	if (fDtrace1 \|\| fDtrace2)
10140	{
10141	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10142	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10143	switch (enmEvent1)
10144	{
10145	/** @todo consider which extra parameters would be helpful for each probe. */
10146	case DBGFEVENT_END: break;
10147	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10148	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10149	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10150	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10151	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10152	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10153	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10154	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10155	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10156	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10157	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10158	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10159	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10160	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10161	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10162	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10163	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10164	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10165	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10166	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10167	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10168	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10169	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10170	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10171	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10172	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10173	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10174	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10175	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10176	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10177	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10178	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10179	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10180	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10181	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10182	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10183	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10184	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10185	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10186	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10187	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10188	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10189	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10190	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10191	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10192	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10193	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10194	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10195	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10196	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10197	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10198	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10199	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10200	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10201	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10202	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10203	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10204	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10205	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10206	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10207	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10208	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10209	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10210	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10211	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10212	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10213	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10214	}
10215	switch (enmEvent2)
10216	{
10217	/** @todo consider which extra parameters would be helpful for each probe. */
10218	case DBGFEVENT_END: break;
10219	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10220	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10221	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10222	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10223	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10224	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10225	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10226	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10227	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10228	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10229	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10230	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10231	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10232	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10233	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10234	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10235	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10236	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10237	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10238	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10239	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10240	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10241	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10242	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10243	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10244	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10245	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10246	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10247	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10248	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10249	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10250	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10251	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10252	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10253	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10254	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10255	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10256	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10257	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10258	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10259	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10260	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10261	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10262	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10263	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10264	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10265	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10266	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10267	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10268	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10269	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10270	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10271	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10272	}
10273	}
10274
10275	/*
10276	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10277	* the DBGF call will do a full check).
10278	*
10279	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10280	* Note! If we have to events, we prioritize the first, i.e. the instruction
10281	* one, in order to avoid event nesting.
10282	*/
10283	if ( enmEvent1 != DBGFEVENT_END
10284	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10285	{
10286	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10287	if (rcStrict != VINF_SUCCESS)
10288	return rcStrict;
10289	}
10290	else if ( enmEvent2 != DBGFEVENT_END
10291	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10292	{
10293	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10294	if (rcStrict != VINF_SUCCESS)
10295	return rcStrict;
10296	}
10297
10298	return VINF_SUCCESS;
10299	}
10300
10301
10302	/**
10303	* Single-stepping VM-exit filtering.
10304	*
10305	* This is preprocessing the VM-exits and deciding whether we've gotten far
10306	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10307	* handling is performed.
10308	*
10309	* @returns Strict VBox status code (i.e. informational status codes too).
10310	* @param pVM The cross context VM structure.
10311	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10312	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10313	* out-of-sync. Make sure to update the required
10314	* fields before using them.
10315	* @param pVmxTransient Pointer to the VMX-transient structure.
10316	* @param uExitReason The VM-exit reason.
10317	* @param pDbgState The debug state.
10318	*/
10319	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10320	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10321	{
10322	/*
10323	* Expensive (saves context) generic dtrace VM-exit probe.
10324	*/
10325	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10326	{ /* more likely */ }
10327	else
10328	{
10329	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10330	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10331	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10332	}
10333
10334	/*
10335	* Check for host NMI, just to get that out of the way.
10336	*/
10337	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10338	{ /* normally likely */ }
10339	else
10340	{
10341	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10342	AssertRCReturn(rc2, rc2);
10343	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10344	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10345	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10346	}
10347
10348	/*
10349	* Check for single stepping event if we're stepping.
10350	*/
10351	if (pVCpu->hm.s.fSingleInstruction)
10352	{
10353	switch (uExitReason)
10354	{
10355	case VMX_EXIT_MTF:
10356	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10357
10358	/* Various events: */
10359	case VMX_EXIT_XCPT_OR_NMI:
10360	case VMX_EXIT_EXT_INT:
10361	case VMX_EXIT_TRIPLE_FAULT:
10362	case VMX_EXIT_INT_WINDOW:
10363	case VMX_EXIT_NMI_WINDOW:
10364	case VMX_EXIT_TASK_SWITCH:
10365	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10366	case VMX_EXIT_APIC_ACCESS:
10367	case VMX_EXIT_EPT_VIOLATION:
10368	case VMX_EXIT_EPT_MISCONFIG:
10369	case VMX_EXIT_PREEMPT_TIMER:
10370
10371	/* Instruction specific VM-exits: */
10372	case VMX_EXIT_CPUID:
10373	case VMX_EXIT_GETSEC:
10374	case VMX_EXIT_HLT:
10375	case VMX_EXIT_INVD:
10376	case VMX_EXIT_INVLPG:
10377	case VMX_EXIT_RDPMC:
10378	case VMX_EXIT_RDTSC:
10379	case VMX_EXIT_RSM:
10380	case VMX_EXIT_VMCALL:
10381	case VMX_EXIT_VMCLEAR:
10382	case VMX_EXIT_VMLAUNCH:
10383	case VMX_EXIT_VMPTRLD:
10384	case VMX_EXIT_VMPTRST:
10385	case VMX_EXIT_VMREAD:
10386	case VMX_EXIT_VMRESUME:
10387	case VMX_EXIT_VMWRITE:
10388	case VMX_EXIT_VMXOFF:
10389	case VMX_EXIT_VMXON:
10390	case VMX_EXIT_MOV_CRX:
10391	case VMX_EXIT_MOV_DRX:
10392	case VMX_EXIT_IO_INSTR:
10393	case VMX_EXIT_RDMSR:
10394	case VMX_EXIT_WRMSR:
10395	case VMX_EXIT_MWAIT:
10396	case VMX_EXIT_MONITOR:
10397	case VMX_EXIT_PAUSE:
10398	case VMX_EXIT_XDTR_ACCESS:
10399	case VMX_EXIT_TR_ACCESS:
10400	case VMX_EXIT_INVEPT:
10401	case VMX_EXIT_RDTSCP:
10402	case VMX_EXIT_INVVPID:
10403	case VMX_EXIT_WBINVD:
10404	case VMX_EXIT_XSETBV:
10405	case VMX_EXIT_RDRAND:
10406	case VMX_EXIT_INVPCID:
10407	case VMX_EXIT_VMFUNC:
10408	case VMX_EXIT_RDSEED:
10409	case VMX_EXIT_XSAVES:
10410	case VMX_EXIT_XRSTORS:
10411	{
10412	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10413	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10414	AssertRCReturn(rc2, rc2);
10415	if ( pMixedCtx->rip != pDbgState->uRipStart
10416	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10417	return VINF_EM_DBG_STEPPED;
10418	break;
10419	}
10420
10421	/* Errors and unexpected events: */
10422	case VMX_EXIT_INIT_SIGNAL:
10423	case VMX_EXIT_SIPI:
10424	case VMX_EXIT_IO_SMI:
10425	case VMX_EXIT_SMI:
10426	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10427	case VMX_EXIT_ERR_MSR_LOAD:
10428	case VMX_EXIT_ERR_MACHINE_CHECK:
10429	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10430	break;
10431
10432	default:
10433	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10434	break;
10435	}
10436	}
10437
10438	/*
10439	* Check for debugger event breakpoints and dtrace probes.
10440	*/
10441	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10442	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10443	{
10444	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10445	if (rcStrict != VINF_SUCCESS)
10446	return rcStrict;
10447	}
10448
10449	/*
10450	* Normal processing.
10451	*/
10452	#ifdef HMVMX_USE_FUNCTION_TABLE
10453	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10454	#else
10455	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10456	#endif
10457	}
10458
10459
10460	/**
10461	* Single steps guest code using VT-x.
10462	*
10463	* @returns Strict VBox status code (i.e. informational status codes too).
10464	* @param pVM The cross context VM structure.
10465	* @param pVCpu The cross context virtual CPU structure.
10466	* @param pCtx Pointer to the guest-CPU context.
10467	*
10468	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10469	*/
10470	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10471	{
10472	VMXTRANSIENT VmxTransient;
10473	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10474
10475	/* Set HMCPU indicators. */
10476	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10477	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10478	pVCpu->hm.s.fDebugWantRdTscExit = false;
10479	pVCpu->hm.s.fUsingDebugLoop = true;
10480
10481	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10482	VMXRUNDBGSTATE DbgState;
10483	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10484	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10485
10486	/*
10487	* The loop.
10488	*/
10489	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10490	for (uint32_t cLoops = 0; ; cLoops++)
10491	{
10492	Assert(!HMR0SuspendPending());
10493	HMVMX_ASSERT_CPU_SAFE();
10494	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10495
10496	/*
10497	* Preparatory work for running guest code, this may force us to return
10498	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10499	*/
10500	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10501	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10502	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10503	if (rcStrict != VINF_SUCCESS)
10504	break;
10505
10506	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10507	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10508
10509	/*
10510	* Now we can run the guest code.
10511	*/
10512	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10513
10514	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10515
10516	/*
10517	* Restore any residual host-state and save any bits shared between host
10518	* and guest into the guest-CPU state. Re-enables interrupts!
10519	*/
10520	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10521
10522	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10523	if (RT_SUCCESS(rcRun))
10524	{ /* very likely */ }
10525	else
10526	{
10527	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10528	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10529	return rcRun;
10530	}
10531
10532	/* Profile the VM-exit. */
10533	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10534	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10535	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10536	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10537	HMVMX_START_EXIT_DISPATCH_PROF();
10538
10539	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10540
10541	/*
10542	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10543	*/
10544	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10545	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10546	if (rcStrict != VINF_SUCCESS)
10547	break;
10548	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10549	{
10550	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10551	rcStrict = VINF_EM_RAW_INTERRUPT;
10552	break;
10553	}
10554
10555	/*
10556	* Stepping: Did the RIP change, if so, consider it a single step.
10557	* Otherwise, make sure one of the TFs gets set.
10558	*/
10559	if (fStepping)
10560	{
10561	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10562	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10563	AssertRCReturn(rc2, rc2);
10564	if ( pCtx->rip != DbgState.uRipStart
10565	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10566	{
10567	rcStrict = VINF_EM_DBG_STEPPED;
10568	break;
10569	}
10570	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10571	}
10572
10573	/*
10574	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10575	*/
10576	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10577	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10578	}
10579
10580	/*
10581	* Clear the X86_EFL_TF if necessary.
10582	*/
10583	if (pVCpu->hm.s.fClearTrapFlag)
10584	{
10585	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10586	AssertRCReturn(rc2, rc2);
10587	pVCpu->hm.s.fClearTrapFlag = false;
10588	pCtx->eflags.Bits.u1TF = 0;
10589	}
10590	/** @todo there seems to be issues with the resume flag when the monitor trap
10591	* flag is pending without being used. Seen early in bios init when
10592	* accessing APIC page in protected mode. */
10593
10594	/*
10595	* Restore VM-exit control settings as we may not reenter this function the
10596	* next time around.
10597	*/
10598	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10599
10600	/* Restore HMCPU indicators. */
10601	pVCpu->hm.s.fUsingDebugLoop = false;
10602	pVCpu->hm.s.fDebugWantRdTscExit = false;
10603	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10604
10605	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10606	return rcStrict;
10607	}
10608
10609
10610	/** @} */
10611
10612
10613	/**
10614	* Checks if any expensive dtrace probes are enabled and we should go to the
10615	* debug loop.
10616	*
10617	* @returns true if we should use debug loop, false if not.
10618	*/
10619	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10620	{
10621	/* It's probably faster to OR the raw 32-bit counter variables together.
10622	Since the variables are in an array and the probes are next to one
10623	another (more or less), we have good locality. So, better read
10624	eight-nine cache lines ever time and only have one conditional, than
10625	128+ conditionals, right? */
10626	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10627	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10628	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10629	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10630	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10631	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10632	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10633	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10634	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10635	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10636	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10637	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10638	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10639	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10640	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10641	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10642	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10643	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10644	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10645	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10646	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10647	) != 0
10648	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10649	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10650	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10651	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10652	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10653	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10654	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10655	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10656	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10657	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10658	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10659	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10660	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10661	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10662	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10663	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10664	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10665	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10666	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10667	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10668	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10669	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10670	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10671	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10672	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10673	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10674	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10675	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10676	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10677	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10678	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10679	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10680	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10681	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10682	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10683	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10684	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10685	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10686	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10687	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10688	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10689	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10690	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10691	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10692	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10693	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10694	) != 0
10695	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10696	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10697	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10698	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10699	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10700	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10701	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10702	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10703	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10704	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10705	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10706	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10707	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10708	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10709	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10710	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10711	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10712	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10713	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10714	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10715	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10716	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10717	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10718	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10719	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10720	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10721	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10722	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10723	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10724	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10725	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10726	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10727	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10728	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10729	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10730	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10731	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10732	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10733	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10734	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10735	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10736	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10737	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10738	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10739	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10740	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10741	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10742	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10743	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10744	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10745	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10746	) != 0;
10747	}
10748
10749
10750	/**
10751	* Runs the guest code using VT-x.
10752	*
10753	* @returns Strict VBox status code (i.e. informational status codes too).
10754	* @param pVM The cross context VM structure.
10755	* @param pVCpu The cross context virtual CPU structure.
10756	* @param pCtx Pointer to the guest-CPU context.
10757	*/
10758	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10759	{
10760	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10761	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10762	HMVMX_ASSERT_PREEMPT_SAFE();
10763
10764	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10765
10766	VBOXSTRICTRC rcStrict;
10767	if ( !pVCpu->hm.s.fUseDebugLoop
10768	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10769	&& !DBGFIsStepping(pVCpu)
10770	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10771	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10772	else
10773	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10774
10775	if (rcStrict == VERR_EM_INTERPRETER)
10776	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10777	else if (rcStrict == VINF_EM_RESET)
10778	rcStrict = VINF_EM_TRIPLE_FAULT;
10779
10780	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10781	if (RT_FAILURE(rc2))
10782	{
10783	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10784	rcStrict = rc2;
10785	}
10786	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10787	return rcStrict;
10788	}
10789
10790
10791	#ifndef HMVMX_USE_FUNCTION_TABLE
10792	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10793	{
10794	# ifdef DEBUG_ramshankar
10795	# define RETURN_EXIT_CALL(a_CallExpr) \
10796	do { \
10797	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10798	VBOXSTRICTRC rcStrict = a_CallExpr; \
10799	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10800	return rcStrict; \
10801	} while (0)
10802	# else
10803	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10804	# endif
10805	switch (rcReason)
10806	{
10807	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10808	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10809	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10810	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10811	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10812	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10813	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10814	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10815	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10816	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10817	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10818	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10819	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10820	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10821	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10822	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10823	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10824	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10825	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10826	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10827	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10828	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10829	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10830	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10831	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10832	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10833	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10834	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10835	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10836	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10837	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10838	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10839	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10840	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10841
10842	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10843	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10844	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10845	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10846	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10847	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10848	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10849	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10850	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10851
10852	case VMX_EXIT_VMCLEAR:
10853	case VMX_EXIT_VMLAUNCH:
10854	case VMX_EXIT_VMPTRLD:
10855	case VMX_EXIT_VMPTRST:
10856	case VMX_EXIT_VMREAD:
10857	case VMX_EXIT_VMRESUME:
10858	case VMX_EXIT_VMWRITE:
10859	case VMX_EXIT_VMXOFF:
10860	case VMX_EXIT_VMXON:
10861	case VMX_EXIT_INVEPT:
10862	case VMX_EXIT_INVVPID:
10863	case VMX_EXIT_VMFUNC:
10864	case VMX_EXIT_XSAVES:
10865	case VMX_EXIT_XRSTORS:
10866	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10867	case VMX_EXIT_ENCLS:
10868	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10869	case VMX_EXIT_PML_FULL:
10870	default:
10871	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10872	}
10873	#undef RETURN_EXIT_CALL
10874	}
10875	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10876
10877
10878	#ifdef VBOX_STRICT
10879	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10880	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10881	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10882
10883	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10884	do { \
10885	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10886	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10887	} while (0)
10888
10889	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10890	do { \
10891	AssertPtr(pVCpu); \
10892	AssertPtr(pMixedCtx); \
10893	AssertPtr(pVmxTransient); \
10894	Assert(pVmxTransient->fVMEntryFailed == false); \
10895	Assert(ASMIntAreEnabled()); \
10896	HMVMX_ASSERT_PREEMPT_SAFE(); \
10897	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10898	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10899	HMVMX_ASSERT_PREEMPT_SAFE(); \
10900	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10901	HMVMX_ASSERT_PREEMPT_CPUID(); \
10902	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10903	} while (0)
10904
10905	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10906	do { \
10907	Log4Func(("\n")); \
10908	} while (0)
10909	#else /* nonstrict builds: */
10910	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10911	do { \
10912	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10913	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10914	} while (0)
10915	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10916	#endif
10917
10918
10919	/**
10920	* Advances the guest RIP by the specified number of bytes.
10921	*
10922	* @param pVCpu The cross context virtual CPU structure.
10923	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10924	* out-of-sync. Make sure to update the required fields
10925	* before using them.
10926	* @param cbInstr Number of bytes to advance the RIP by.
10927	*
10928	* @remarks No-long-jump zone!!!
10929	*/
10930	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10931	{
10932	/* Advance the RIP. */
10933	pMixedCtx->rip += cbInstr;
10934	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10935
10936	/* Update interrupt inhibition. */
10937	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10938	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10939	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10940	}
10941
10942
10943	/**
10944	* Advances the guest RIP after reading it from the VMCS.
10945	*
10946	* @returns VBox status code, no informational status codes.
10947	* @param pVCpu The cross context virtual CPU structure.
10948	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10949	* out-of-sync. Make sure to update the required fields
10950	* before using them.
10951	* @param pVmxTransient Pointer to the VMX transient structure.
10952	*
10953	* @remarks No-long-jump zone!!!
10954	*/
10955	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10956	{
10957	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10958	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10959	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10960	AssertRCReturn(rc, rc);
10961
10962	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10963
10964	/*
10965	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10966	* pending debug exception field as it takes care of priority of events.
10967	*
10968	* See Intel spec. 32.2.1 "Debug Exceptions".
10969	*/
10970	if ( !pVCpu->hm.s.fSingleInstruction
10971	&& pMixedCtx->eflags.Bits.u1TF)
10972	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10973
10974	return VINF_SUCCESS;
10975	}
10976
10977
10978	/**
10979	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10980	* and update error record fields accordingly.
10981	*
10982	* @return VMX_IGS_* return codes.
10983	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10984	* wrong with the guest state.
10985	*
10986	* @param pVM The cross context VM structure.
10987	* @param pVCpu The cross context virtual CPU structure.
10988	* @param pCtx Pointer to the guest-CPU state.
10989	*
10990	* @remarks This function assumes our cache of the VMCS controls
10991	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10992	*/
10993	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10994	{
10995	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10996	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10997	uError = (err); \
10998	break; \
10999	} else do { } while (0)
11000
11001	int rc;
11002	uint32_t uError = VMX_IGS_ERROR;
11003	uint32_t u32Val;
11004	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
11005
11006	do
11007	{
11008	/*
11009	* CR0.
11010	*/
11011	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11012	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11013	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
11014	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
11015	if (fUnrestrictedGuest)
11016	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
11017
11018	uint32_t u32GuestCR0;
11019	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
11020	AssertRCBreak(rc);
11021	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
11022	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
11023	if ( !fUnrestrictedGuest
11024	&& (u32GuestCR0 & X86_CR0_PG)
11025	&& !(u32GuestCR0 & X86_CR0_PE))
11026	{
11027	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
11028	}
11029
11030	/*
11031	* CR4.
11032	*/
11033	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11034	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11035
11036	uint32_t u32GuestCR4;
11037	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11038	AssertRCBreak(rc);
11039	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11040	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11041
11042	/*
11043	* IA32_DEBUGCTL MSR.
11044	*/
11045	uint64_t u64Val;
11046	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11047	AssertRCBreak(rc);
11048	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11049	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11050	{
11051	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11052	}
11053	uint64_t u64DebugCtlMsr = u64Val;
11054
11055	#ifdef VBOX_STRICT
11056	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11057	AssertRCBreak(rc);
11058	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11059	#endif
11060	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11061
11062	/*
11063	* RIP and RFLAGS.
11064	*/
11065	uint32_t u32Eflags;
11066	#if HC_ARCH_BITS == 64
11067	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11068	AssertRCBreak(rc);
11069	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11070	if ( !fLongModeGuest
11071	\|\| !pCtx->cs.Attr.n.u1Long)
11072	{
11073	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11074	}
11075	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11076	* must be identical if the "IA-32e mode guest" VM-entry
11077	* control is 1 and CS.L is 1. No check applies if the
11078	* CPU supports 64 linear-address bits. */
11079
11080	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11081	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11082	AssertRCBreak(rc);
11083	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11084	VMX_IGS_RFLAGS_RESERVED);
11085	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11086	u32Eflags = u64Val;
11087	#else
11088	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11089	AssertRCBreak(rc);
11090	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11091	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11092	#endif
11093
11094	if ( fLongModeGuest
11095	\|\| ( fUnrestrictedGuest
11096	&& !(u32GuestCR0 & X86_CR0_PE)))
11097	{
11098	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11099	}
11100
11101	uint32_t u32EntryInfo;
11102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11103	AssertRCBreak(rc);
11104	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11105	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11106	{
11107	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11108	}
11109
11110	/*
11111	* 64-bit checks.
11112	*/
11113	#if HC_ARCH_BITS == 64
11114	if (fLongModeGuest)
11115	{
11116	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11117	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11118	}
11119
11120	if ( !fLongModeGuest
11121	&& (u32GuestCR4 & X86_CR4_PCIDE))
11122	{
11123	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11124	}
11125
11126	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11127	* 51:32 beyond the processor's physical-address width are 0. */
11128
11129	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11130	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11131	{
11132	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11133	}
11134
11135	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11136	AssertRCBreak(rc);
11137	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11138
11139	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11140	AssertRCBreak(rc);
11141	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11142	#endif
11143
11144	/*
11145	* PERF_GLOBAL MSR.
11146	*/
11147	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11148	{
11149	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11150	AssertRCBreak(rc);
11151	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11152	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11153	}
11154
11155	/*
11156	* PAT MSR.
11157	*/
11158	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11159	{
11160	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11161	AssertRCBreak(rc);
11162	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11163	for (unsigned i = 0; i < 8; i++)
11164	{
11165	uint8_t u8Val = (u64Val & 0xff);
11166	if ( u8Val != 0 /* UC */
11167	&& u8Val != 1 /* WC */
11168	&& u8Val != 4 /* WT */
11169	&& u8Val != 5 /* WP */
11170	&& u8Val != 6 /* WB */
11171	&& u8Val != 7 /* UC- */)
11172	{
11173	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11174	}
11175	u64Val >>= 8;
11176	}
11177	}
11178
11179	/*
11180	* EFER MSR.
11181	*/
11182	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11183	{
11184	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11185	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11186	AssertRCBreak(rc);
11187	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11188	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11189	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11190	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11191	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11192	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11193	\|\| !(u32GuestCR0 & X86_CR0_PG)
11194	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11195	VMX_IGS_EFER_LMA_LME_MISMATCH);
11196	}
11197
11198	/*
11199	* Segment registers.
11200	*/
11201	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11202	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11203	if (!(u32Eflags & X86_EFL_VM))
11204	{
11205	/* CS */
11206	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11207	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11208	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11209	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11210	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11211	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11212	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11213	/* CS cannot be loaded with NULL in protected mode. */
11214	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11215	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11216	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11217	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11218	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11219	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11220	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11221	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11222	else
11223	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11224
11225	/* SS */
11226	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11227	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11228	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11229	if ( !(pCtx->cr0 & X86_CR0_PE)
11230	\|\| pCtx->cs.Attr.n.u4Type == 3)
11231	{
11232	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11233	}
11234	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11235	{
11236	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11237	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11238	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11239	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11240	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11241	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11242	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11243	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11244	}
11245
11246	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11247	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11248	{
11249	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11250	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11251	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11252	\|\| pCtx->ds.Attr.n.u4Type > 11
11253	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11254	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11255	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11256	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11257	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11258	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11259	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11260	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11261	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11262	}
11263	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11264	{
11265	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11266	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11267	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11268	\|\| pCtx->es.Attr.n.u4Type > 11
11269	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11270	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11271	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11272	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11273	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11274	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11275	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11276	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11277	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11278	}
11279	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11280	{
11281	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11282	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11283	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11284	\|\| pCtx->fs.Attr.n.u4Type > 11
11285	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11286	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11287	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11288	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11289	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11290	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11291	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11292	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11293	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11294	}
11295	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11296	{
11297	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11298	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11299	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11300	\|\| pCtx->gs.Attr.n.u4Type > 11
11301	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11302	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11303	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11304	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11305	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11306	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11307	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11308	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11309	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11310	}
11311	/* 64-bit capable CPUs. */
11312	#if HC_ARCH_BITS == 64
11313	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11314	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11315	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11316	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11317	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11318	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11319	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11320	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11321	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11322	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11323	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11324	#endif
11325	}
11326	else
11327	{
11328	/* V86 mode checks. */
11329	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11330	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11331	{
11332	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11333	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11334	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11335	}
11336	else
11337	{
11338	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11339	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11340	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11341	}
11342
11343	/* CS */
11344	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11345	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11346	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11347	/* SS */
11348	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11349	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11350	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11351	/* DS */
11352	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11353	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11354	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11355	/* ES */
11356	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11357	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11358	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11359	/* FS */
11360	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11361	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11362	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11363	/* GS */
11364	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11365	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11366	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11367	/* 64-bit capable CPUs. */
11368	#if HC_ARCH_BITS == 64
11369	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11370	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11371	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11372	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11373	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11374	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11375	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11376	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11377	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11378	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11379	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11380	#endif
11381	}
11382
11383	/*
11384	* TR.
11385	*/
11386	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11387	/* 64-bit capable CPUs. */
11388	#if HC_ARCH_BITS == 64
11389	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11390	#endif
11391	if (fLongModeGuest)
11392	{
11393	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11394	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11395	}
11396	else
11397	{
11398	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11399	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11400	VMX_IGS_TR_ATTR_TYPE_INVALID);
11401	}
11402	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11403	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11404	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11405	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11406	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11407	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11408	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11409	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11410
11411	/*
11412	* GDTR and IDTR.
11413	*/
11414	#if HC_ARCH_BITS == 64
11415	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11416	AssertRCBreak(rc);
11417	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11418
11419	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11420	AssertRCBreak(rc);
11421	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11422	#endif
11423
11424	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11425	AssertRCBreak(rc);
11426	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11427
11428	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11429	AssertRCBreak(rc);
11430	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11431
11432	/*
11433	* Guest Non-Register State.
11434	*/
11435	/* Activity State. */
11436	uint32_t u32ActivityState;
11437	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11438	AssertRCBreak(rc);
11439	HMVMX_CHECK_BREAK( !u32ActivityState
11440	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11441	VMX_IGS_ACTIVITY_STATE_INVALID);
11442	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11443	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11444	uint32_t u32IntrState;
11445	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11446	AssertRCBreak(rc);
11447	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11448	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11449	{
11450	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11451	}
11452
11453	/** @todo Activity state and injecting interrupts. Left as a todo since we
11454	* currently don't use activity states but ACTIVE. */
11455
11456	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11457	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11458
11459	/* Guest interruptibility-state. */
11460	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11461	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11462	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11463	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11464	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11465	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11466	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11467	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11468	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11469	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11470	{
11471	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11472	{
11473	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11474	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11475	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11476	}
11477	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11478	{
11479	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11480	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11481	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11482	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11483	}
11484	}
11485	/** @todo Assumes the processor is not in SMM. */
11486	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11487	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11488	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11489	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11490	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11491	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11492	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11493	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11494	{
11495	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11496	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11497	}
11498
11499	/* Pending debug exceptions. */
11500	#if HC_ARCH_BITS == 64
11501	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11502	AssertRCBreak(rc);
11503	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11504	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11505	u32Val = u64Val; /* For pending debug exceptions checks below. */
11506	#else
11507	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11508	AssertRCBreak(rc);
11509	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11510	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11511	#endif
11512
11513	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11514	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11515	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11516	{
11517	if ( (u32Eflags & X86_EFL_TF)
11518	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11519	{
11520	/* Bit 14 is PendingDebug.BS. */
11521	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11522	}
11523	if ( !(u32Eflags & X86_EFL_TF)
11524	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11525	{
11526	/* Bit 14 is PendingDebug.BS. */
11527	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11528	}
11529	}
11530
11531	/* VMCS link pointer. */
11532	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11533	AssertRCBreak(rc);
11534	if (u64Val != UINT64_C(0xffffffffffffffff))
11535	{
11536	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11537	/** @todo Bits beyond the processor's physical-address width MBZ. */
11538	/** @todo 32-bit located in memory referenced by value of this field (as a
11539	* physical address) must contain the processor's VMCS revision ID. */
11540	/** @todo SMM checks. */
11541	}
11542
11543	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11544	* not using Nested Paging? */
11545	if ( pVM->hm.s.fNestedPaging
11546	&& !fLongModeGuest
11547	&& CPUMIsGuestInPAEModeEx(pCtx))
11548	{
11549	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11550	AssertRCBreak(rc);
11551	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11552
11553	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11554	AssertRCBreak(rc);
11555	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11556
11557	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11558	AssertRCBreak(rc);
11559	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11560
11561	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11562	AssertRCBreak(rc);
11563	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11564	}
11565
11566	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11567	if (uError == VMX_IGS_ERROR)
11568	uError = VMX_IGS_REASON_NOT_FOUND;
11569	} while (0);
11570
11571	pVCpu->hm.s.u32HMError = uError;
11572	return uError;
11573
11574	#undef HMVMX_ERROR_BREAK
11575	#undef HMVMX_CHECK_BREAK
11576	}
11577
11578	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11579	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11580	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11581
11582	/** @name VM-exit handlers.
11583	* @{
11584	*/
11585
11586	/**
11587	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11588	*/
11589	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11590	{
11591	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11593	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11594	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11595	return VINF_SUCCESS;
11596	return VINF_EM_RAW_INTERRUPT;
11597	}
11598
11599
11600	/**
11601	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11602	*/
11603	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11604	{
11605	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11606	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11607
11608	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11609	AssertRCReturn(rc, rc);
11610
11611	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11612	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11613	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11614	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11615
11616	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11617	{
11618	/*
11619	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11620	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11621	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11622	*
11623	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11624	*/
11625	VMXDispatchHostNmi();
11626	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11627	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11628	return VINF_SUCCESS;
11629	}
11630
11631	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11632	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11633	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11634	{ /* likely */ }
11635	else
11636	{
11637	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11638	rcStrictRc1 = VINF_SUCCESS;
11639	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11640	return rcStrictRc1;
11641	}
11642
11643	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11644	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11645	switch (uIntType)
11646	{
11647	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11648	Assert(uVector == X86_XCPT_DB);
11649	RT_FALL_THRU();
11650	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11651	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11652	RT_FALL_THRU();
11653	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11654	{
11655	/*
11656	* If there's any exception caused as a result of event injection, the resulting
11657	* secondary/final execption will be pending, we shall continue guest execution
11658	* after injecting the event. The page-fault case is complicated and we manually
11659	* handle any currently pending event in hmR0VmxExitXcptPF.
11660	*/
11661	if (!pVCpu->hm.s.Event.fPending)
11662	{ /* likely */ }
11663	else if (uVector != X86_XCPT_PF)
11664	{
11665	rc = VINF_SUCCESS;
11666	break;
11667	}
11668
11669	switch (uVector)
11670	{
11671	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11672	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11673	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11674	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11675	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11676	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11677	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11678
11679	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11680	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11681	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11682	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11683	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11684	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11685	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11686	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11687	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11688	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11689	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11690	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11691	default:
11692	{
11693	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11694	AssertRCReturn(rc, rc);
11695
11696	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11697	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11698	{
11699	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11700	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11701	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11702
11703	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11704	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11705	AssertRCReturn(rc, rc);
11706	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11707	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11708	0 /* GCPtrFaultAddress */);
11709	AssertRCReturn(rc, rc);
11710	}
11711	else
11712	{
11713	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11714	pVCpu->hm.s.u32HMError = uVector;
11715	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11716	}
11717	break;
11718	}
11719	}
11720	break;
11721	}
11722
11723	default:
11724	{
11725	pVCpu->hm.s.u32HMError = uExitIntInfo;
11726	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11727	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11728	break;
11729	}
11730	}
11731	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11732	return rc;
11733	}
11734
11735
11736	/**
11737	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11738	*/
11739	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11740	{
11741	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11742
11743	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11744	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11745
11746	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11747	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11748	return VINF_SUCCESS;
11749	}
11750
11751
11752	/**
11753	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11754	*/
11755	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11756	{
11757	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11758	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11759	{
11760	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11761	HMVMX_RETURN_UNEXPECTED_EXIT();
11762	}
11763
11764	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11765
11766	/*
11767	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11768	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11769	*/
11770	uint32_t uIntrState = 0;
11771	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11772	AssertRCReturn(rc, rc);
11773
11774	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11775	if ( fBlockSti
11776	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11777	{
11778	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11779	}
11780
11781	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11782	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11783
11784	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11785	return VINF_SUCCESS;
11786	}
11787
11788
11789	/**
11790	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11791	*/
11792	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11793	{
11794	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11796	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11797	}
11798
11799
11800	/**
11801	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11802	*/
11803	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11804	{
11805	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11806	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11807	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11808	}
11809
11810
11811	/**
11812	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11813	*/
11814	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11815	{
11816	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11817	PVM pVM = pVCpu->CTX_SUFF(pVM);
11818	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11819	if (RT_LIKELY(rc == VINF_SUCCESS))
11820	{
11821	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11822	Assert(pVmxTransient->cbInstr == 2);
11823	}
11824	else
11825	{
11826	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11827	rc = VERR_EM_INTERPRETER;
11828	}
11829	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11830	return rc;
11831	}
11832
11833
11834	/**
11835	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11836	*/
11837	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11838	{
11839	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11840	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11841	AssertRCReturn(rc, rc);
11842
11843	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11844	return VINF_EM_RAW_EMULATE_INSTR;
11845
11846	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11847	HMVMX_RETURN_UNEXPECTED_EXIT();
11848	}
11849
11850
11851	/**
11852	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11853	*/
11854	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11855	{
11856	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11857	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11858	AssertRCReturn(rc, rc);
11859
11860	PVM pVM = pVCpu->CTX_SUFF(pVM);
11861	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11862	if (RT_LIKELY(rc == VINF_SUCCESS))
11863	{
11864	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11865	Assert(pVmxTransient->cbInstr == 2);
11866	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11867	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11868	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11869	}
11870	else
11871	rc = VERR_EM_INTERPRETER;
11872	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11873	return rc;
11874	}
11875
11876
11877	/**
11878	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11879	*/
11880	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11881	{
11882	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11883	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11884	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11885	AssertRCReturn(rc, rc);
11886
11887	PVM pVM = pVCpu->CTX_SUFF(pVM);
11888	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11889	if (RT_SUCCESS(rc))
11890	{
11891	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11892	Assert(pVmxTransient->cbInstr == 3);
11893	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11894	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11895	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11896	}
11897	else
11898	{
11899	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11900	rc = VERR_EM_INTERPRETER;
11901	}
11902	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11903	return rc;
11904	}
11905
11906
11907	/**
11908	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11909	*/
11910	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11911	{
11912	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11913	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11914	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11915	AssertRCReturn(rc, rc);
11916
11917	PVM pVM = pVCpu->CTX_SUFF(pVM);
11918	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11919	if (RT_LIKELY(rc == VINF_SUCCESS))
11920	{
11921	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11922	Assert(pVmxTransient->cbInstr == 2);
11923	}
11924	else
11925	{
11926	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11927	rc = VERR_EM_INTERPRETER;
11928	}
11929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11930	return rc;
11931	}
11932
11933
11934	/**
11935	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11936	*/
11937	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11938	{
11939	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11940	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11941
11942	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11943	if (pVCpu->hm.s.fHypercallsEnabled)
11944	{
11945	#if 0
11946	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11947	#else
11948	/* Aggressive state sync. for now. */
11949	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11950	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11951	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11952	AssertRCReturn(rc, rc);
11953	#endif
11954
11955	/* Perform the hypercall. */
11956	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11957	if (rcStrict == VINF_SUCCESS)
11958	{
11959	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11960	AssertRCReturn(rc, rc);
11961	}
11962	else
11963	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11964	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11965	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11966
11967	/* If the hypercall changes anything other than guest's general-purpose registers,
11968	we would need to reload the guest changed bits here before VM-entry. */
11969	}
11970	else
11971	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11972
11973	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11974	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11975	{
11976	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11977	rcStrict = VINF_SUCCESS;
11978	}
11979
11980	return rcStrict;
11981	}
11982
11983
11984	/**
11985	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11986	*/
11987	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11988	{
11989	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11990	PVM pVM = pVCpu->CTX_SUFF(pVM);
11991	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11992
11993	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11994	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11995	AssertRCReturn(rc, rc);
11996
11997	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11998	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11999	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12000	else
12001	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
12002	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
12003	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
12004	return rcStrict;
12005	}
12006
12007
12008	/**
12009	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
12010	*/
12011	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12012	{
12013	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12014	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12015	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12016	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12017	AssertRCReturn(rc, rc);
12018
12019	PVM pVM = pVCpu->CTX_SUFF(pVM);
12020	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12021	if (RT_LIKELY(rc == VINF_SUCCESS))
12022	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12023	else
12024	{
12025	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
12026	rc = VERR_EM_INTERPRETER;
12027	}
12028	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
12029	return rc;
12030	}
12031
12032
12033	/**
12034	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12035	*/
12036	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12037	{
12038	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12039	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12040	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12041	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12042	AssertRCReturn(rc, rc);
12043
12044	PVM pVM = pVCpu->CTX_SUFF(pVM);
12045	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12046	rc = VBOXSTRICTRC_VAL(rc2);
12047	if (RT_LIKELY( rc == VINF_SUCCESS
12048	\|\| rc == VINF_EM_HALT))
12049	{
12050	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12051	AssertRCReturn(rc3, rc3);
12052
12053	if ( rc == VINF_EM_HALT
12054	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12055	{
12056	rc = VINF_SUCCESS;
12057	}
12058	}
12059	else
12060	{
12061	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12062	rc = VERR_EM_INTERPRETER;
12063	}
12064	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
12065	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12066	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12067	return rc;
12068	}
12069
12070
12071	/**
12072	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12073	*/
12074	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12075	{
12076	/*
12077	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12078	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12079	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
12080	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12081	*/
12082	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12083	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12084	HMVMX_RETURN_UNEXPECTED_EXIT();
12085	}
12086
12087
12088	/**
12089	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12090	*/
12091	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12092	{
12093	/*
12094	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12095	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12096	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12097	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12098	*/
12099	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12100	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12101	HMVMX_RETURN_UNEXPECTED_EXIT();
12102	}
12103
12104
12105	/**
12106	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12107	*/
12108	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12109	{
12110	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12111	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12112	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12113	HMVMX_RETURN_UNEXPECTED_EXIT();
12114	}
12115
12116
12117	/**
12118	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12119	*/
12120	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12121	{
12122	/*
12123	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12124	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12125	* See Intel spec. 25.3 "Other Causes of VM-exits".
12126	*/
12127	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12128	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12129	HMVMX_RETURN_UNEXPECTED_EXIT();
12130	}
12131
12132
12133	/**
12134	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12135	* VM-exit.
12136	*/
12137	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12138	{
12139	/*
12140	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12141	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12142	*
12143	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12144	* See Intel spec. "23.8 Restrictions on VMX operation".
12145	*/
12146	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12147	return VINF_SUCCESS;
12148	}
12149
12150
12151	/**
12152	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12153	* VM-exit.
12154	*/
12155	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12156	{
12157	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12158	return VINF_EM_RESET;
12159	}
12160
12161
12162	/**
12163	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12164	*/
12165	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12166	{
12167	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12168	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12169
12170	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12171	AssertRCReturn(rc, rc);
12172
12173	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12174	rc = VINF_SUCCESS;
12175	else
12176	rc = VINF_EM_HALT;
12177
12178	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12179	if (rc != VINF_SUCCESS)
12180	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12181	return rc;
12182	}
12183
12184
12185	/**
12186	* VM-exit handler for instructions that result in a \#UD exception delivered to
12187	* the guest.
12188	*/
12189	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12190	{
12191	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12192	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12193	return VINF_SUCCESS;
12194	}
12195
12196
12197	/**
12198	* VM-exit handler for expiry of the VMX preemption timer.
12199	*/
12200	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12201	{
12202	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12203
12204	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12205	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12206
12207	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12208	PVM pVM = pVCpu->CTX_SUFF(pVM);
12209	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12210	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12211	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12212	}
12213
12214
12215	/**
12216	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12217	*/
12218	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12219	{
12220	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12221
12222	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12223	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12224	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12225	AssertRCReturn(rc, rc);
12226
12227	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12228	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12229
12230	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12231
12232	return rcStrict;
12233	}
12234
12235
12236	/**
12237	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12238	*/
12239	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12240	{
12241	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12242	/** @todo Use VM-exit instruction information. */
12243	return VERR_EM_INTERPRETER;
12244	}
12245
12246
12247	/**
12248	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12249	* Error VM-exit.
12250	*/
12251	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12252	{
12253	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12254	AssertRCReturn(rc, rc);
12255
12256	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12257	AssertRCReturn(rc, rc);
12258
12259	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12260	NOREF(uInvalidReason);
12261
12262	#ifdef VBOX_STRICT
12263	uint32_t uIntrState;
12264	RTHCUINTREG uHCReg;
12265	uint64_t u64Val;
12266	uint32_t u32Val;
12267
12268	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12269	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12270	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12271	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12272	AssertRCReturn(rc, rc);
12273
12274	Log4(("uInvalidReason %u\n", uInvalidReason));
12275	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12276	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12277	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12278	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12279
12280	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12281	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12282	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12283	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12284	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12285	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12286	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12287	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12288	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12289	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12290	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12291	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12292	#else
12293	NOREF(pVmxTransient);
12294	#endif
12295
12296	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12297	return VERR_VMX_INVALID_GUEST_STATE;
12298	}
12299
12300
12301	/**
12302	* VM-exit handler for VM-entry failure due to an MSR-load
12303	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12304	*/
12305	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12306	{
12307	NOREF(pVmxTransient);
12308	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12309	HMVMX_RETURN_UNEXPECTED_EXIT();
12310	}
12311
12312
12313	/**
12314	* VM-exit handler for VM-entry failure due to a machine-check event
12315	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12316	*/
12317	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12318	{
12319	NOREF(pVmxTransient);
12320	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12321	HMVMX_RETURN_UNEXPECTED_EXIT();
12322	}
12323
12324
12325	/**
12326	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12327	* theory.
12328	*/
12329	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12330	{
12331	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12332	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12333	return VERR_VMX_UNDEFINED_EXIT_CODE;
12334	}
12335
12336
12337	/**
12338	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12339	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12340	* Conditional VM-exit.
12341	*/
12342	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12343	{
12344	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12345
12346	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12347	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12348	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12349	return VERR_EM_INTERPRETER;
12350	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12351	HMVMX_RETURN_UNEXPECTED_EXIT();
12352	}
12353
12354
12355	/**
12356	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12357	*/
12358	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12359	{
12360	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12361
12362	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12363	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12364	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12365	return VERR_EM_INTERPRETER;
12366	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12367	HMVMX_RETURN_UNEXPECTED_EXIT();
12368	}
12369
12370
12371	/**
12372	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12373	*/
12374	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12375	{
12376	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12377
12378	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12379	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12380	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12381	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12382	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12383	{
12384	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12385	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12386	}
12387	AssertRCReturn(rc, rc);
12388	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12389
12390	#ifdef VBOX_STRICT
12391	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12392	{
12393	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12394	&& pMixedCtx->ecx != MSR_K6_EFER)
12395	{
12396	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12397	pMixedCtx->ecx));
12398	HMVMX_RETURN_UNEXPECTED_EXIT();
12399	}
12400	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12401	{
12402	VMXMSREXITREAD enmRead;
12403	VMXMSREXITWRITE enmWrite;
12404	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12405	AssertRCReturn(rc2, rc2);
12406	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12407	{
12408	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12409	HMVMX_RETURN_UNEXPECTED_EXIT();
12410	}
12411	}
12412	}
12413	#endif
12414
12415	PVM pVM = pVCpu->CTX_SUFF(pVM);
12416	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12417	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12418	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12419	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12420	if (RT_SUCCESS(rc))
12421	{
12422	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12423	Assert(pVmxTransient->cbInstr == 2);
12424	}
12425	return rc;
12426	}
12427
12428
12429	/**
12430	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12431	*/
12432	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12433	{
12434	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12435	PVM pVM = pVCpu->CTX_SUFF(pVM);
12436	int rc = VINF_SUCCESS;
12437
12438	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12439	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12440	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12441	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12442	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12443	{
12444	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12445	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12446	}
12447	AssertRCReturn(rc, rc);
12448	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12449
12450	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12451	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12452	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12453
12454	if (RT_SUCCESS(rc))
12455	{
12456	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12457
12458	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12459	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12460	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12461	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12462	{
12463	/*
12464	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12465	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12466	* EMInterpretWrmsr() changes it.
12467	*/
12468	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12469	}
12470	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12471	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12472	else if (pMixedCtx->ecx == MSR_K6_EFER)
12473	{
12474	/*
12475	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12476	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12477	* the other bits as well, SCE and NXE. See @bugref{7368}.
12478	*/
12479	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12480	}
12481
12482	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12483	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12484	{
12485	switch (pMixedCtx->ecx)
12486	{
12487	/*
12488	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12489	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12490	*/
12491	case MSR_IA32_SYSENTER_CS:
12492	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12493	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12494	break;
12495	case MSR_IA32_SYSENTER_EIP:
12496	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12497	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12498	break;
12499	case MSR_IA32_SYSENTER_ESP:
12500	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12501	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12502	break;
12503	case MSR_K8_FS_BASE: RT_FALL_THRU();
12504	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12505	case MSR_K6_EFER: /* already handled above */ break;
12506	default:
12507	{
12508	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12509	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12510	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12511	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12512	break;
12513	}
12514	}
12515	}
12516	#ifdef VBOX_STRICT
12517	else
12518	{
12519	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12520	switch (pMixedCtx->ecx)
12521	{
12522	case MSR_IA32_SYSENTER_CS:
12523	case MSR_IA32_SYSENTER_EIP:
12524	case MSR_IA32_SYSENTER_ESP:
12525	case MSR_K8_FS_BASE:
12526	case MSR_K8_GS_BASE:
12527	{
12528	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12529	HMVMX_RETURN_UNEXPECTED_EXIT();
12530	}
12531
12532	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12533	default:
12534	{
12535	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12536	{
12537	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12538	if (pMixedCtx->ecx != MSR_K6_EFER)
12539	{
12540	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12541	pMixedCtx->ecx));
12542	HMVMX_RETURN_UNEXPECTED_EXIT();
12543	}
12544	}
12545
12546	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12547	{
12548	VMXMSREXITREAD enmRead;
12549	VMXMSREXITWRITE enmWrite;
12550	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12551	AssertRCReturn(rc2, rc2);
12552	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12553	{
12554	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12555	HMVMX_RETURN_UNEXPECTED_EXIT();
12556	}
12557	}
12558	break;
12559	}
12560	}
12561	}
12562	#endif /* VBOX_STRICT */
12563	}
12564	return rc;
12565	}
12566
12567
12568	/**
12569	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12570	*/
12571	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12572	{
12573	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12574
12575	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12576	return VINF_EM_RAW_INTERRUPT;
12577	}
12578
12579
12580	/**
12581	* VM-exit handler for when the TPR value is lowered below the specified
12582	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12583	*/
12584	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12585	{
12586	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12587	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12588
12589	/*
12590	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12591	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12592	*/
12593	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12594	return VINF_SUCCESS;
12595	}
12596
12597
12598	/**
12599	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12600	* VM-exit.
12601	*
12602	* @retval VINF_SUCCESS when guest execution can continue.
12603	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12604	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12605	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12606	* interpreter.
12607	*/
12608	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12609	{
12610	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12611	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12612	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12613	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12614	AssertRCReturn(rc, rc);
12615
12616	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12617	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12618	PVM pVM = pVCpu->CTX_SUFF(pVM);
12619	VBOXSTRICTRC rcStrict;
12620	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12621	switch (uAccessType)
12622	{
12623	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12624	{
12625	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12626	AssertRCReturn(rc, rc);
12627
12628	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12629	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12630	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12631	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12632	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12633	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12634	{
12635	case 0: /* CR0 */
12636	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12637	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12638	break;
12639	case 2: /* CR2 */
12640	/* Nothing to do here, CR2 it's not part of the VMCS. */
12641	break;
12642	case 3: /* CR3 */
12643	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12644	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12645	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12646	break;
12647	case 4: /* CR4 */
12648	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12649	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12650	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12651	break;
12652	case 8: /* CR8 */
12653	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12654	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12655	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12656	break;
12657	default:
12658	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12659	break;
12660	}
12661
12662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12663	break;
12664	}
12665
12666	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12667	{
12668	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12669	AssertRCReturn(rc, rc);
12670
12671	Assert( !pVM->hm.s.fNestedPaging
12672	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12673	\|\| pVCpu->hm.s.fUsingDebugLoop
12674	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12675
12676	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12677	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12678	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12679
12680	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12681	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12682	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12683	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12684	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12685	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12686	VBOXSTRICTRC_VAL(rcStrict)));
12687	if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12688	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
12689	break;
12690	}
12691
12692	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12693	{
12694	AssertRCReturn(rc, rc);
12695	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12696	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12697	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12698	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12699	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12700	break;
12701	}
12702
12703	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12704	{
12705	AssertRCReturn(rc, rc);
12706	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12707	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12708	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12709	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12710	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12711	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12712	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12713	break;
12714	}
12715
12716	default:
12717	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12718	VERR_VMX_UNEXPECTED_EXCEPTION);
12719	}
12720
12721	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12722	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12723	NOREF(pVM);
12724	return rcStrict;
12725	}
12726
12727
12728	/**
12729	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12730	* VM-exit.
12731	*/
12732	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12733	{
12734	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12735	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12736
12737	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12738	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12739	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12740	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12741	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12742	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12743	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12744	AssertRCReturn(rc, rc);
12745
12746	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12747	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12748	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12749	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12750	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12751	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12752	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12753	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12754	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12755
12756	/* I/O operation lookup arrays. */
12757	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12758	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12759
12760	VBOXSTRICTRC rcStrict;
12761	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12762	uint32_t const cbInstr = pVmxTransient->cbInstr;
12763	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12764	PVM pVM = pVCpu->CTX_SUFF(pVM);
12765	if (fIOString)
12766	{
12767	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12768	See @bugref{5752#c158}. Should work now. */
12769	/*
12770	* INS/OUTS - I/O String instruction.
12771	*
12772	* Use instruction-information if available, otherwise fall back on
12773	* interpreting the instruction.
12774	*/
12775	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12776	fIOWrite ? 'w' : 'r'));
12777	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12778	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12779	{
12780	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12781	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12782	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12783	AssertRCReturn(rc2, rc2);
12784	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12785	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12786	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12787	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12788	if (fIOWrite)
12789	{
12790	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12791	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12792	}
12793	else
12794	{
12795	/*
12796	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12797	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12798	* See Intel Instruction spec. for "INS".
12799	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12800	*/
12801	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12802	}
12803	}
12804	else
12805	{
12806	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12807	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12808	AssertRCReturn(rc2, rc2);
12809	rcStrict = IEMExecOne(pVCpu);
12810	}
12811	/** @todo IEM needs to be setting these flags somehow. */
12812	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12813	fUpdateRipAlready = true;
12814	#else
12815	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12816	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12817	if (RT_SUCCESS(rcStrict))
12818	{
12819	if (fIOWrite)
12820	{
12821	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12822	(DISCPUMODE)pDis->uAddrMode, cbValue);
12823	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12824	}
12825	else
12826	{
12827	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12828	(DISCPUMODE)pDis->uAddrMode, cbValue);
12829	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12830	}
12831	}
12832	else
12833	{
12834	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12835	pMixedCtx->rip));
12836	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12837	}
12838	#endif
12839	}
12840	else
12841	{
12842	/*
12843	* IN/OUT - I/O instruction.
12844	*/
12845	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12846	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12847	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12848	if (fIOWrite)
12849	{
12850	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12851	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12852	}
12853	else
12854	{
12855	uint32_t u32Result = 0;
12856	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12857	if (IOM_SUCCESS(rcStrict))
12858	{
12859	/* Save result of I/O IN instr. in AL/AX/EAX. */
12860	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12861	}
12862	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12863	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12864	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12865	}
12866	}
12867
12868	if (IOM_SUCCESS(rcStrict))
12869	{
12870	if (!fUpdateRipAlready)
12871	{
12872	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12873	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12874	}
12875
12876	/*
12877	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12878	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12879	*/
12880	if (fIOString)
12881	{
12882	/** @todo Single-step for INS/OUTS with REP prefix? */
12883	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12884	}
12885	else if ( !fDbgStepping
12886	&& fGstStepping)
12887	{
12888	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12889	}
12890
12891	/*
12892	* If any I/O breakpoints are armed, we need to check if one triggered
12893	* and take appropriate action.
12894	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12895	*/
12896	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12897	AssertRCReturn(rc2, rc2);
12898
12899	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12900	* execution engines about whether hyper BPs and such are pending. */
12901	uint32_t const uDr7 = pMixedCtx->dr[7];
12902	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12903	&& X86_DR7_ANY_RW_IO(uDr7)
12904	&& (pMixedCtx->cr4 & X86_CR4_DE))
12905	\|\| DBGFBpIsHwIoArmed(pVM)))
12906	{
12907	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12908
12909	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12910	VMMRZCallRing3Disable(pVCpu);
12911	HM_DISABLE_PREEMPT();
12912
12913	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12914
12915	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12916	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12917	{
12918	/* Raise #DB. */
12919	if (fIsGuestDbgActive)
12920	ASMSetDR6(pMixedCtx->dr[6]);
12921	if (pMixedCtx->dr[7] != uDr7)
12922	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12923
12924	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12925	}
12926	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12927	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12928	else if ( rcStrict2 != VINF_SUCCESS
12929	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12930	rcStrict = rcStrict2;
12931	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12932
12933	HM_RESTORE_PREEMPT();
12934	VMMRZCallRing3Enable(pVCpu);
12935	}
12936	}
12937
12938	#ifdef VBOX_STRICT
12939	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12940	Assert(!fIOWrite);
12941	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12942	Assert(fIOWrite);
12943	else
12944	{
12945	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12946	* statuses, that the VMM device and some others may return. See
12947	* IOM_SUCCESS() for guidance. */
12948	AssertMsg( RT_FAILURE(rcStrict)
12949	\|\| rcStrict == VINF_SUCCESS
12950	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12951	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12952	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12953	\|\| rcStrict == VINF_EM_RAW_TO_R3
12954	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12955	#endif
12956	}
12957	#endif
12958
12959	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12960	return rcStrict;
12961	}
12962
12963
12964	/**
12965	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12966	* VM-exit.
12967	*/
12968	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12969	{
12970	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12971
12972	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12973	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12974	AssertRCReturn(rc, rc);
12975	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12976	{
12977	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12978	AssertRCReturn(rc, rc);
12979	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12980	{
12981	uint32_t uErrCode;
12982	RTGCUINTPTR GCPtrFaultAddress;
12983	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12984	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12985	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12986	if (fErrorCodeValid)
12987	{
12988	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12989	AssertRCReturn(rc, rc);
12990	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12991	}
12992	else
12993	uErrCode = 0;
12994
12995	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12996	&& uVector == X86_XCPT_PF)
12997	GCPtrFaultAddress = pMixedCtx->cr2;
12998	else
12999	GCPtrFaultAddress = 0;
13000
13001	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13002	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
13003
13004	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
13005	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13006	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13007	}
13008	}
13009
13010	/* Fall back to the interpreter to emulate the task-switch. */
13011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13012	return VERR_EM_INTERPRETER;
13013	}
13014
13015
13016	/**
13017	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
13018	*/
13019	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13020	{
13021	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13022	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
13023	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
13024	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13025	AssertRCReturn(rc, rc);
13026	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
13027	return VINF_EM_DBG_STEPPED;
13028	}
13029
13030
13031	/**
13032	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
13033	*/
13034	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13035	{
13036	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13037
13038	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
13039
13040	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13041	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13042	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13043	{
13044	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
13045	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13046	{
13047	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13048	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13049	}
13050	}
13051	else
13052	{
13053	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13054	rcStrict1 = VINF_SUCCESS;
13055	return rcStrict1;
13056	}
13057
13058	#if 0
13059	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13060	* just sync the whole thing. */
13061	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13062	#else
13063	/* Aggressive state sync. for now. */
13064	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13065	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13066	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13067	#endif
13068	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13069	AssertRCReturn(rc, rc);
13070
13071	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13072	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13073	VBOXSTRICTRC rcStrict2;
13074	switch (uAccessType)
13075	{
13076	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13077	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13078	{
13079	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13080	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13081	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13082
13083	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13084	GCPhys &= PAGE_BASE_GC_MASK;
13085	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13086	PVM pVM = pVCpu->CTX_SUFF(pVM);
13087	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13088	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13089
13090	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13091	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13092	CPUMCTX2CORE(pMixedCtx), GCPhys);
13093	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13094	if ( rcStrict2 == VINF_SUCCESS
13095	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13096	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13097	{
13098	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13099	\| HM_CHANGED_GUEST_RSP
13100	\| HM_CHANGED_GUEST_RFLAGS
13101	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13102	rcStrict2 = VINF_SUCCESS;
13103	}
13104	break;
13105	}
13106
13107	default:
13108	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13109	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13110	break;
13111	}
13112
13113	if (rcStrict2 != VINF_SUCCESS)
13114	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13115	return rcStrict2;
13116	}
13117
13118
13119	/**
13120	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13121	* VM-exit.
13122	*/
13123	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13124	{
13125	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13126
13127	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13128	if (pVmxTransient->fWasGuestDebugStateActive)
13129	{
13130	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13131	HMVMX_RETURN_UNEXPECTED_EXIT();
13132	}
13133
13134	if ( !pVCpu->hm.s.fSingleInstruction
13135	&& !pVmxTransient->fWasHyperDebugStateActive)
13136	{
13137	Assert(!DBGFIsStepping(pVCpu));
13138	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13139
13140	/* Don't intercept MOV DRx any more. */
13141	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13142	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13143	AssertRCReturn(rc, rc);
13144
13145	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13146	VMMRZCallRing3Disable(pVCpu);
13147	HM_DISABLE_PREEMPT();
13148
13149	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13150	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13151	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13152
13153	HM_RESTORE_PREEMPT();
13154	VMMRZCallRing3Enable(pVCpu);
13155
13156	#ifdef VBOX_WITH_STATISTICS
13157	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13158	AssertRCReturn(rc, rc);
13159	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13160	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13161	else
13162	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13163	#endif
13164	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13165	return VINF_SUCCESS;
13166	}
13167
13168	/*
13169	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13170	* Update the segment registers and DR7 from the CPU.
13171	*/
13172	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13173	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13174	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13175	AssertRCReturn(rc, rc);
13176	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13177
13178	PVM pVM = pVCpu->CTX_SUFF(pVM);
13179	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13180	{
13181	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13182	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13183	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13184	if (RT_SUCCESS(rc))
13185	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13187	}
13188	else
13189	{
13190	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13191	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13192	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13193	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13194	}
13195
13196	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13197	if (RT_SUCCESS(rc))
13198	{
13199	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13200	AssertRCReturn(rc2, rc2);
13201	return VINF_SUCCESS;
13202	}
13203	return rc;
13204	}
13205
13206
13207	/**
13208	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13209	* Conditional VM-exit.
13210	*/
13211	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13212	{
13213	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13214	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13215
13216	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13217	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13218	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13219	{
13220	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13221	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13222	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13223	{
13224	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13225	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13226	}
13227	}
13228	else
13229	{
13230	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13231	rcStrict1 = VINF_SUCCESS;
13232	return rcStrict1;
13233	}
13234
13235	RTGCPHYS GCPhys = 0;
13236	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13237
13238	#if 0
13239	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13240	#else
13241	/* Aggressive state sync. for now. */
13242	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13243	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13244	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13245	#endif
13246	AssertRCReturn(rc, rc);
13247
13248	/*
13249	* If we succeed, resume guest execution.
13250	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13251	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13252	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13253	* weird case. See @bugref{6043}.
13254	*/
13255	PVM pVM = pVCpu->CTX_SUFF(pVM);
13256	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13257	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13258	if ( rcStrict2 == VINF_SUCCESS
13259	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13260	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13261	{
13262	/* Successfully handled MMIO operation. */
13263	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13264	\| HM_CHANGED_GUEST_RSP
13265	\| HM_CHANGED_GUEST_RFLAGS
13266	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13267	return VINF_SUCCESS;
13268	}
13269	return rcStrict2;
13270	}
13271
13272
13273	/**
13274	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13275	* VM-exit.
13276	*/
13277	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13278	{
13279	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13280	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13281
13282	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13283	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13284	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13285	{
13286	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13287	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13288	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13289	}
13290	else
13291	{
13292	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13293	rcStrict1 = VINF_SUCCESS;
13294	return rcStrict1;
13295	}
13296
13297	RTGCPHYS GCPhys = 0;
13298	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13299	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13300	#if 0
13301	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13302	#else
13303	/* Aggressive state sync. for now. */
13304	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13305	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13306	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13307	#endif
13308	AssertRCReturn(rc, rc);
13309
13310	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13311	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13312
13313	RTGCUINT uErrorCode = 0;
13314	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13315	uErrorCode \|= X86_TRAP_PF_ID;
13316	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13317	uErrorCode \|= X86_TRAP_PF_RW;
13318	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13319	uErrorCode \|= X86_TRAP_PF_P;
13320
13321	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13322
13323	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13324	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13325
13326	/* Handle the pagefault trap for the nested shadow table. */
13327	PVM pVM = pVCpu->CTX_SUFF(pVM);
13328	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13329	TRPMResetTrap(pVCpu);
13330
13331	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13332	if ( rcStrict2 == VINF_SUCCESS
13333	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13334	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13335	{
13336	/* Successfully synced our nested page tables. */
13337	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13338	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13339	\| HM_CHANGED_GUEST_RSP
13340	\| HM_CHANGED_GUEST_RFLAGS);
13341	return VINF_SUCCESS;
13342	}
13343
13344	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13345	return rcStrict2;
13346	}
13347
13348	/** @} */
13349
13350	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13351	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13352	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13353
13354	/** @name VM-exit exception handlers.
13355	* @{
13356	*/
13357
13358	/**
13359	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13360	*/
13361	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13362	{
13363	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13364	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13365
13366	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13367	AssertRCReturn(rc, rc);
13368
13369	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13370	{
13371	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13372	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13373
13374	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13375	* provides VM-exit instruction length. If this causes problem later,
13376	* disassemble the instruction like it's done on AMD-V. */
13377	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13378	AssertRCReturn(rc2, rc2);
13379	return rc;
13380	}
13381
13382	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13383	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13384	return rc;
13385	}
13386
13387
13388	/**
13389	* VM-exit exception handler for \#BP (Breakpoint exception).
13390	*/
13391	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13392	{
13393	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13394	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13395
13396	/** @todo Try optimize this by not saving the entire guest state unless
13397	* really needed. */
13398	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13399	AssertRCReturn(rc, rc);
13400
13401	PVM pVM = pVCpu->CTX_SUFF(pVM);
13402	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13403	if (rc == VINF_EM_RAW_GUEST_TRAP)
13404	{
13405	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13406	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13407	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13408	AssertRCReturn(rc, rc);
13409
13410	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13411	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13412	}
13413
13414	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13415	return rc;
13416	}
13417
13418
13419	/**
13420	* VM-exit exception handler for \#AC (alignment check exception).
13421	*/
13422	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13423	{
13424	RT_NOREF_PV(pMixedCtx);
13425	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13426
13427	/*
13428	* Re-inject it. We'll detect any nesting before getting here.
13429	*/
13430	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13431	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13432	AssertRCReturn(rc, rc);
13433	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13434
13435	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13436	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13437	return VINF_SUCCESS;
13438	}
13439
13440
13441	/**
13442	* VM-exit exception handler for \#DB (Debug exception).
13443	*/
13444	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13445	{
13446	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13447	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13448	Log6(("XcptDB\n"));
13449
13450	/*
13451	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13452	* for processing.
13453	*/
13454	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13455	AssertRCReturn(rc, rc);
13456
13457	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13458	uint64_t uDR6 = X86_DR6_INIT_VAL;
13459	uDR6 \|= ( pVmxTransient->uExitQualification
13460	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13461
13462	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13463	if (rc == VINF_EM_RAW_GUEST_TRAP)
13464	{
13465	/*
13466	* The exception was for the guest. Update DR6, DR7.GD and
13467	* IA32_DEBUGCTL.LBR before forwarding it.
13468	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13469	*/
13470	VMMRZCallRing3Disable(pVCpu);
13471	HM_DISABLE_PREEMPT();
13472
13473	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13474	pMixedCtx->dr[6] \|= uDR6;
13475	if (CPUMIsGuestDebugStateActive(pVCpu))
13476	ASMSetDR6(pMixedCtx->dr[6]);
13477
13478	HM_RESTORE_PREEMPT();
13479	VMMRZCallRing3Enable(pVCpu);
13480
13481	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13482	AssertRCReturn(rc, rc);
13483
13484	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13485	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13486
13487	/* Paranoia. */
13488	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13489	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13490
13491	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13492	AssertRCReturn(rc, rc);
13493
13494	/*
13495	* Raise #DB in the guest.
13496	*
13497	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13498	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13499	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13500	*
13501	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13502	*/
13503	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13504	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13505	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13506	AssertRCReturn(rc, rc);
13507	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13508	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13509	return VINF_SUCCESS;
13510	}
13511
13512	/*
13513	* Not a guest trap, must be a hypervisor related debug event then.
13514	* Update DR6 in case someone is interested in it.
13515	*/
13516	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13517	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13518	CPUMSetHyperDR6(pVCpu, uDR6);
13519
13520	return rc;
13521	}
13522
13523
13524	/**
13525	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13526	* point exception).
13527	*/
13528	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13529	{
13530	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13531
13532	/* We require CR0 and EFER. EFER is always up-to-date. */
13533	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13534	AssertRCReturn(rc, rc);
13535
13536	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13537	VMMRZCallRing3Disable(pVCpu);
13538	HM_DISABLE_PREEMPT();
13539
13540	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13541	if (pVmxTransient->fWasGuestFPUStateActive)
13542	{
13543	rc = VINF_EM_RAW_GUEST_TRAP;
13544	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13545	}
13546	else
13547	{
13548	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13549	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13550	#endif
13551	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13552	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13553	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13554	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13555	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13556	}
13557
13558	HM_RESTORE_PREEMPT();
13559	VMMRZCallRing3Enable(pVCpu);
13560
13561	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13562	{
13563	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13564	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13565	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13566	pVCpu->hm.s.fPreloadGuestFpu = true;
13567	}
13568	else
13569	{
13570	/* Forward #NM to the guest. */
13571	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13572	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13573	AssertRCReturn(rc, rc);
13574	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13575	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13576	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13577	}
13578
13579	return VINF_SUCCESS;
13580	}
13581
13582
13583	/**
13584	* VM-exit exception handler for \#GP (General-protection exception).
13585	*
13586	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13587	*/
13588	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13589	{
13590	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13592
13593	int rc;
13594	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13595	{ /* likely */ }
13596	else
13597	{
13598	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13599	Assert(pVCpu->hm.s.fUsingDebugLoop);
13600	#endif
13601	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13602	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13603	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13604	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13605	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13606	AssertRCReturn(rc, rc);
13607	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13608	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13609	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13610	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13611	return rc;
13612	}
13613
13614	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13615	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13616
13617	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13618	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13619	AssertRCReturn(rc, rc);
13620
13621	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13622	uint32_t cbOp = 0;
13623	PVM pVM = pVCpu->CTX_SUFF(pVM);
13624	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13625	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13626	if (RT_SUCCESS(rc))
13627	{
13628	rc = VINF_SUCCESS;
13629	Assert(cbOp == pDis->cbInstr);
13630	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13631	switch (pDis->pCurInstr->uOpcode)
13632	{
13633	case OP_CLI:
13634	{
13635	pMixedCtx->eflags.Bits.u1IF = 0;
13636	pMixedCtx->eflags.Bits.u1RF = 0;
13637	pMixedCtx->rip += pDis->cbInstr;
13638	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13639	if ( !fDbgStepping
13640	&& pMixedCtx->eflags.Bits.u1TF)
13641	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13642	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13643	break;
13644	}
13645
13646	case OP_STI:
13647	{
13648	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13649	pMixedCtx->eflags.Bits.u1IF = 1;
13650	pMixedCtx->eflags.Bits.u1RF = 0;
13651	pMixedCtx->rip += pDis->cbInstr;
13652	if (!fOldIF)
13653	{
13654	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13655	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13656	}
13657	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13658	if ( !fDbgStepping
13659	&& pMixedCtx->eflags.Bits.u1TF)
13660	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13661	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13662	break;
13663	}
13664
13665	case OP_HLT:
13666	{
13667	rc = VINF_EM_HALT;
13668	pMixedCtx->rip += pDis->cbInstr;
13669	pMixedCtx->eflags.Bits.u1RF = 0;
13670	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13671	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13672	break;
13673	}
13674
13675	case OP_POPF:
13676	{
13677	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13678	uint32_t cbParm;
13679	uint32_t uMask;
13680	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13681	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13682	{
13683	cbParm = 4;
13684	uMask = 0xffffffff;
13685	}
13686	else
13687	{
13688	cbParm = 2;
13689	uMask = 0xffff;
13690	}
13691
13692	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13693	RTGCPTR GCPtrStack = 0;
13694	X86EFLAGS Eflags;
13695	Eflags.u32 = 0;
13696	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13697	&GCPtrStack);
13698	if (RT_SUCCESS(rc))
13699	{
13700	Assert(sizeof(Eflags.u32) >= cbParm);
13701	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13702	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13703	}
13704	if (RT_FAILURE(rc))
13705	{
13706	rc = VERR_EM_INTERPRETER;
13707	break;
13708	}
13709	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13710	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13711	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13712	pMixedCtx->esp += cbParm;
13713	pMixedCtx->esp &= uMask;
13714	pMixedCtx->rip += pDis->cbInstr;
13715	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13716	\| HM_CHANGED_GUEST_RSP
13717	\| HM_CHANGED_GUEST_RFLAGS);
13718	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13719	POPF restores EFLAGS.TF. */
13720	if ( !fDbgStepping
13721	&& fGstStepping)
13722	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13724	break;
13725	}
13726
13727	case OP_PUSHF:
13728	{
13729	uint32_t cbParm;
13730	uint32_t uMask;
13731	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13732	{
13733	cbParm = 4;
13734	uMask = 0xffffffff;
13735	}
13736	else
13737	{
13738	cbParm = 2;
13739	uMask = 0xffff;
13740	}
13741
13742	/* Get the stack pointer & push the contents of eflags onto the stack. */
13743	RTGCPTR GCPtrStack = 0;
13744	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13745	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13746	if (RT_FAILURE(rc))
13747	{
13748	rc = VERR_EM_INTERPRETER;
13749	break;
13750	}
13751	X86EFLAGS Eflags = pMixedCtx->eflags;
13752	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13753	Eflags.Bits.u1RF = 0;
13754	Eflags.Bits.u1VM = 0;
13755
13756	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13757	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13758	{
13759	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13760	rc = VERR_EM_INTERPRETER;
13761	break;
13762	}
13763	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13764	pMixedCtx->esp -= cbParm;
13765	pMixedCtx->esp &= uMask;
13766	pMixedCtx->rip += pDis->cbInstr;
13767	pMixedCtx->eflags.Bits.u1RF = 0;
13768	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13769	\| HM_CHANGED_GUEST_RSP
13770	\| HM_CHANGED_GUEST_RFLAGS);
13771	if ( !fDbgStepping
13772	&& pMixedCtx->eflags.Bits.u1TF)
13773	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13774	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13775	break;
13776	}
13777
13778	case OP_IRET:
13779	{
13780	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13781	* instruction reference. */
13782	RTGCPTR GCPtrStack = 0;
13783	uint32_t uMask = 0xffff;
13784	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13785	uint16_t aIretFrame[3];
13786	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13787	{
13788	rc = VERR_EM_INTERPRETER;
13789	break;
13790	}
13791	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13792	&GCPtrStack);
13793	if (RT_SUCCESS(rc))
13794	{
13795	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13796	PGMACCESSORIGIN_HM));
13797	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13798	}
13799	if (RT_FAILURE(rc))
13800	{
13801	rc = VERR_EM_INTERPRETER;
13802	break;
13803	}
13804	pMixedCtx->eip = 0;
13805	pMixedCtx->ip = aIretFrame[0];
13806	pMixedCtx->cs.Sel = aIretFrame[1];
13807	pMixedCtx->cs.ValidSel = aIretFrame[1];
13808	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13809	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13810	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13811	pMixedCtx->sp += sizeof(aIretFrame);
13812	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13813	\| HM_CHANGED_GUEST_SEGMENT_REGS
13814	\| HM_CHANGED_GUEST_RSP
13815	\| HM_CHANGED_GUEST_RFLAGS);
13816	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13817	if ( !fDbgStepping
13818	&& fGstStepping)
13819	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13820	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13821	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13822	break;
13823	}
13824
13825	case OP_INT:
13826	{
13827	uint16_t uVector = pDis->Param1.uValue & 0xff;
13828	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13829	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13831	break;
13832	}
13833
13834	case OP_INTO:
13835	{
13836	if (pMixedCtx->eflags.Bits.u1OF)
13837	{
13838	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13839	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13840	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13841	}
13842	else
13843	{
13844	pMixedCtx->eflags.Bits.u1RF = 0;
13845	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13846	}
13847	break;
13848	}
13849
13850	default:
13851	{
13852	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13853	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13854	EMCODETYPE_SUPERVISOR);
13855	rc = VBOXSTRICTRC_VAL(rc2);
13856	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13857	/** @todo We have to set pending-debug exceptions here when the guest is
13858	* single-stepping depending on the instruction that was interpreted. */
13859	Log4(("#GP rc=%Rrc\n", rc));
13860	break;
13861	}
13862	}
13863	}
13864	else
13865	rc = VERR_EM_INTERPRETER;
13866
13867	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13868	("#GP Unexpected rc=%Rrc\n", rc));
13869	return rc;
13870	}
13871
13872
13873	/**
13874	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13875	* the exception reported in the VMX transient structure back into the VM.
13876	*
13877	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13878	* up-to-date.
13879	*/
13880	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13881	{
13882	RT_NOREF_PV(pMixedCtx);
13883	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13884	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13885	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13886	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13887	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13888	#endif
13889
13890	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13891	hmR0VmxCheckExitDueToEventDelivery(). */
13892	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13893	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13894	AssertRCReturn(rc, rc);
13895	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13896
13897	#ifdef DEBUG_ramshankar
13898	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13899	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13900	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13901	#endif
13902
13903	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13904	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13905	return VINF_SUCCESS;
13906	}
13907
13908
13909	/**
13910	* VM-exit exception handler for \#PF (Page-fault exception).
13911	*/
13912	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13913	{
13914	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13915	PVM pVM = pVCpu->CTX_SUFF(pVM);
13916	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13917	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13918	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13919	AssertRCReturn(rc, rc);
13920
13921	if (!pVM->hm.s.fNestedPaging)
13922	{ /* likely */ }
13923	else
13924	{
13925	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13926	Assert(pVCpu->hm.s.fUsingDebugLoop);
13927	#endif
13928	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13929	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13930	{
13931	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13932	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13933	}
13934	else
13935	{
13936	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13937	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13938	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13939	}
13940	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13941	return rc;
13942	}
13943
13944	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13945	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13946	if (pVmxTransient->fVectoringPF)
13947	{
13948	Assert(pVCpu->hm.s.Event.fPending);
13949	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13950	}
13951
13952	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13953	AssertRCReturn(rc, rc);
13954
13955	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13956	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13957
13958	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13959	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13960	(RTGCPTR)pVmxTransient->uExitQualification);
13961
13962	Log4(("#PF: rc=%Rrc\n", rc));
13963	if (rc == VINF_SUCCESS)
13964	{
13965	#if 0
13966	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13967	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13968	* memory? We don't update the whole state here... */
13969	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13970	\| HM_CHANGED_GUEST_RSP
13971	\| HM_CHANGED_GUEST_RFLAGS
13972	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13973	#else
13974	/*
13975	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13976	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13977	*/
13978	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13979	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13980	#endif
13981	TRPMResetTrap(pVCpu);
13982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13983	return rc;
13984	}
13985
13986	if (rc == VINF_EM_RAW_GUEST_TRAP)
13987	{
13988	if (!pVmxTransient->fVectoringDoublePF)
13989	{
13990	/* It's a guest page fault and needs to be reflected to the guest. */
13991	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13992	TRPMResetTrap(pVCpu);
13993	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13994	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13995	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13996	}
13997	else
13998	{
13999	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14000	TRPMResetTrap(pVCpu);
14001	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14002	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
14003	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
14004	}
14005
14006	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14007	return VINF_SUCCESS;
14008	}
14009
14010	TRPMResetTrap(pVCpu);
14011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14012	return rc;
14013	}
14014
14015	/** @} */
14016

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 71881

Download in other formats: