HMVMXR0.cpp@ 67116

Last change on this file since 67116 was 67050, checked in by vboxsync, 8 years ago
VMM/APIC, HMVMXR0: Fixes for when the APIC isn't present.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 597.2 KB

Line
1	/* $Id: HMVMXR0.cpp 67050 2017-05-24 05:44:32Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2016 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#define HMVMX_USE_IEM_EVENT_REFLECTION
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50	# define HMVMX_ALWAYS_TRAP_PF
51	# define HMVMX_ALWAYS_SWAP_FPU_STATE
52	# define HMVMX_ALWAYS_FLUSH_TLB
53	# define HMVMX_ALWAYS_SWAP_EFER
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DR7 \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_INTR_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Exception bitmap mask for real-mode guests (real-on-v86).
140	*
141	* We need to intercept all exceptions manually except:
142	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144	* due to bugs in Intel CPUs.
145	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
146	* support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*********************************************************************************************************************************
197	* Structures and Typedefs *
198	*********************************************************************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG fEFlags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit code qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u7Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringDoublePF;
278	/** Whether the VM-exit was caused by a page-fault during delivery of an
279	* external interrupt or NMI. */
280	bool fVectoringPF;
281	} VMXTRANSIENT;
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
285	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
286	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
287	/** Pointer to VMX transient state. */
288	typedef VMXTRANSIENT *PVMXTRANSIENT;
289
290
291	/**
292	* MSR-bitmap read permissions.
293	*/
294	typedef enum VMXMSREXITREAD
295	{
296	/** Reading this MSR causes a VM-exit. */
297	VMXMSREXIT_INTERCEPT_READ = 0xb,
298	/** Reading this MSR does not cause a VM-exit. */
299	VMXMSREXIT_PASSTHRU_READ
300	} VMXMSREXITREAD;
301	/** Pointer to MSR-bitmap read permissions. */
302	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
303
304	/**
305	* MSR-bitmap write permissions.
306	*/
307	typedef enum VMXMSREXITWRITE
308	{
309	/** Writing to this MSR causes a VM-exit. */
310	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
311	/** Writing to this MSR does not cause a VM-exit. */
312	VMXMSREXIT_PASSTHRU_WRITE
313	} VMXMSREXITWRITE;
314	/** Pointer to MSR-bitmap write permissions. */
315	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
316
317
318	/**
319	* VMX VM-exit handler.
320	*
321	* @returns Strict VBox status code (i.e. informational status codes too).
322	* @param pVCpu The cross context virtual CPU structure.
323	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
324	* out-of-sync. Make sure to update the required
325	* fields before using them.
326	* @param pVmxTransient Pointer to the VMX-transient structure.
327	*/
328	#ifndef HMVMX_USE_FUNCTION_TABLE
329	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330	#else
331	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
332	/** Pointer to VM-exit handler. */
333	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
334	#endif
335
336	/**
337	* VMX VM-exit handler, non-strict status code.
338	*
339	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
340	*
341	* @returns VBox status code, no informational status code returned.
342	* @param pVCpu The cross context virtual CPU structure.
343	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
344	* out-of-sync. Make sure to update the required
345	* fields before using them.
346	* @param pVmxTransient Pointer to the VMX-transient structure.
347	*
348	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
349	* use of that status code will be replaced with VINF_EM_SOMETHING
350	* later when switching over to IEM.
351	*/
352	#ifndef HMVMX_USE_FUNCTION_TABLE
353	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
354	#else
355	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
356	#endif
357
358
359	/*********************************************************************************************************************************
360	* Internal Functions *
361	*********************************************************************************************************************************/
362	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
363	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
364	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
365	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
366	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
367	bool fStepping, uint32_t *puIntState);
368	#if HC_ARCH_BITS == 32
369	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
370	#endif
371	#ifndef HMVMX_USE_FUNCTION_TABLE
372	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
373	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
374	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
375	#else
376	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
377	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
378	#endif
379
380
381	/** @name VM-exit handlers.
382	* @{
383	*/
384	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
385	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
386	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
392	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
393	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
394	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
395	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
396	static FNVMXEXITHANDLER hmR0VmxExitHlt;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
398	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
399	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
400	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
404	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
405	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
406	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
407	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
408	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
412	static FNVMXEXITHANDLER hmR0VmxExitMwait;
413	static FNVMXEXITHANDLER hmR0VmxExitMtf;
414	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
415	static FNVMXEXITHANDLER hmR0VmxExitPause;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
417	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
418	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
419	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
422	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
423	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
424	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
425	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
426	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
427	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
428	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
429	/** @} */
430
431	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
440
441
442	/*********************************************************************************************************************************
443	* Global Variables *
444	*********************************************************************************************************************************/
445	#ifdef HMVMX_USE_FUNCTION_TABLE
446
447	/**
448	* VMX_EXIT dispatch table.
449	*/
450	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
451	{
452	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
453	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
454	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
455	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
456	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
457	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
458	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
459	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
460	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
461	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
462	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
463	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
464	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
465	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
466	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
467	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
468	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
469	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
470	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
471	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
472	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
473	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
474	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
475	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
476	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
477	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
478	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
479	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
480	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
481	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
482	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
483	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
484	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
485	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
486	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
487	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
488	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
489	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
490	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
491	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
492	/* 40 UNDEFINED */ hmR0VmxExitPause,
493	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
494	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
495	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
496	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
497	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
498	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
499	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
500	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
501	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
502	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
503	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
504	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
505	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
506	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
507	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
508	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
509	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
510	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
511	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
512	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
513	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
514	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
515	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
516	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
517	};
518	#endif /* HMVMX_USE_FUNCTION_TABLE */
519
520	#ifdef VBOX_STRICT
521	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
522	{
523	/* 0 */ "(Not Used)",
524	/* 1 */ "VMCALL executed in VMX root operation.",
525	/* 2 */ "VMCLEAR with invalid physical address.",
526	/* 3 */ "VMCLEAR with VMXON pointer.",
527	/* 4 */ "VMLAUNCH with non-clear VMCS.",
528	/* 5 */ "VMRESUME with non-launched VMCS.",
529	/* 6 */ "VMRESUME after VMXOFF",
530	/* 7 */ "VM-entry with invalid control fields.",
531	/* 8 */ "VM-entry with invalid host state fields.",
532	/* 9 */ "VMPTRLD with invalid physical address.",
533	/* 10 */ "VMPTRLD with VMXON pointer.",
534	/* 11 */ "VMPTRLD with incorrect revision identifier.",
535	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
536	/* 13 */ "VMWRITE to read-only VMCS component.",
537	/* 14 */ "(Not Used)",
538	/* 15 */ "VMXON executed in VMX root operation.",
539	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
540	/* 17 */ "VM-entry with non-launched executing VMCS.",
541	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
542	/* 19 */ "VMCALL with non-clear VMCS.",
543	/* 20 */ "VMCALL with invalid VM-exit control fields.",
544	/* 21 */ "(Not Used)",
545	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
546	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
547	/* 24 */ "VMCALL with invalid SMM-monitor features.",
548	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
549	/* 26 */ "VM-entry with events blocked by MOV SS.",
550	/* 27 */ "(Not Used)",
551	/* 28 */ "Invalid operand to INVEPT/INVVPID."
552	};
553	#endif /* VBOX_STRICT */
554
555
556
557	/**
558	* Updates the VM's last error record.
559	*
560	* If there was a VMX instruction error, reads the error data from the VMCS and
561	* updates VCPU's last error record as well.
562	*
563	* @param pVM The cross context VM structure.
564	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
565	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
566	* VERR_VMX_INVALID_VMCS_FIELD.
567	* @param rc The error code.
568	*/
569	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
570	{
571	AssertPtr(pVM);
572	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
573	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
574	{
575	AssertPtrReturnVoid(pVCpu);
576	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
577	}
578	pVM->hm.s.lLastError = rc;
579	}
580
581
582	/**
583	* Reads the VM-entry interruption-information field from the VMCS into the VMX
584	* transient structure.
585	*
586	* @returns VBox status code.
587	* @param pVmxTransient Pointer to the VMX transient structure.
588	*
589	* @remarks No-long-jump zone!!!
590	*/
591	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
592	{
593	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
594	AssertRCReturn(rc, rc);
595	return VINF_SUCCESS;
596	}
597
598
599	#ifdef VBOX_STRICT
600	/**
601	* Reads the VM-entry exception error code field from the VMCS into
602	* the VMX transient structure.
603	*
604	* @returns VBox status code.
605	* @param pVmxTransient Pointer to the VMX transient structure.
606	*
607	* @remarks No-long-jump zone!!!
608	*/
609	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610	{
611	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612	AssertRCReturn(rc, rc);
613	return VINF_SUCCESS;
614	}
615	#endif /* VBOX_STRICT */
616
617
618	#ifdef VBOX_STRICT
619	/**
620	* Reads the VM-entry exception error code field from the VMCS into
621	* the VMX transient structure.
622	*
623	* @returns VBox status code.
624	* @param pVmxTransient Pointer to the VMX transient structure.
625	*
626	* @remarks No-long-jump zone!!!
627	*/
628	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
629	{
630	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
631	AssertRCReturn(rc, rc);
632	return VINF_SUCCESS;
633	}
634	#endif /* VBOX_STRICT */
635
636
637	/**
638	* Reads the VM-exit interruption-information field from the VMCS into the VMX
639	* transient structure.
640	*
641	* @returns VBox status code.
642	* @param pVmxTransient Pointer to the VMX transient structure.
643	*/
644	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
645	{
646	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
647	{
648	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
649	AssertRCReturn(rc, rc);
650	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
651	}
652	return VINF_SUCCESS;
653	}
654
655
656	/**
657	* Reads the VM-exit interruption error code from the VMCS into the VMX
658	* transient structure.
659	*
660	* @returns VBox status code.
661	* @param pVmxTransient Pointer to the VMX transient structure.
662	*/
663	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
664	{
665	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
666	{
667	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
668	AssertRCReturn(rc, rc);
669	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
670	}
671	return VINF_SUCCESS;
672	}
673
674
675	/**
676	* Reads the VM-exit instruction length field from the VMCS into the VMX
677	* transient structure.
678	*
679	* @returns VBox status code.
680	* @param pVmxTransient Pointer to the VMX transient structure.
681	*/
682	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
683	{
684	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
685	{
686	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
687	AssertRCReturn(rc, rc);
688	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
689	}
690	return VINF_SUCCESS;
691	}
692
693
694	/**
695	* Reads the VM-exit instruction-information field from the VMCS into
696	* the VMX transient structure.
697	*
698	* @returns VBox status code.
699	* @param pVmxTransient Pointer to the VMX transient structure.
700	*/
701	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
702	{
703	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
704	{
705	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
706	AssertRCReturn(rc, rc);
707	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
708	}
709	return VINF_SUCCESS;
710	}
711
712
713	/**
714	* Reads the exit code qualification from the VMCS into the VMX transient
715	* structure.
716	*
717	* @returns VBox status code.
718	* @param pVCpu The cross context virtual CPU structure of the
719	* calling EMT. (Required for the VMCS cache case.)
720	* @param pVmxTransient Pointer to the VMX transient structure.
721	*/
722	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
723	{
724	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
725	{
726	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
727	AssertRCReturn(rc, rc);
728	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
729	}
730	return VINF_SUCCESS;
731	}
732
733
734	/**
735	* Reads the IDT-vectoring information field from the VMCS into the VMX
736	* transient structure.
737	*
738	* @returns VBox status code.
739	* @param pVmxTransient Pointer to the VMX transient structure.
740	*
741	* @remarks No-long-jump zone!!!
742	*/
743	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
744	{
745	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
746	{
747	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
748	AssertRCReturn(rc, rc);
749	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
750	}
751	return VINF_SUCCESS;
752	}
753
754
755	/**
756	* Reads the IDT-vectoring error code from the VMCS into the VMX
757	* transient structure.
758	*
759	* @returns VBox status code.
760	* @param pVmxTransient Pointer to the VMX transient structure.
761	*/
762	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
763	{
764	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
765	{
766	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
767	AssertRCReturn(rc, rc);
768	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
769	}
770	return VINF_SUCCESS;
771	}
772
773
774	/**
775	* Enters VMX root mode operation on the current CPU.
776	*
777	* @returns VBox status code.
778	* @param pVM The cross context VM structure. Can be
779	* NULL, after a resume.
780	* @param HCPhysCpuPage Physical address of the VMXON region.
781	* @param pvCpuPage Pointer to the VMXON region.
782	*/
783	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
784	{
785	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
786	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
787	Assert(pvCpuPage);
788	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
789
790	if (pVM)
791	{
792	/* Write the VMCS revision dword to the VMXON region. */
793	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
794	}
795
796	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
797	RTCCUINTREG fEFlags = ASMIntDisableFlags();
798
799	/* Enable the VMX bit in CR4 if necessary. */
800	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
801
802	/* Enter VMX root mode. */
803	int rc = VMXEnable(HCPhysCpuPage);
804	if (RT_FAILURE(rc))
805	{
806	if (!(uOldCr4 & X86_CR4_VMXE))
807	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
808
809	if (pVM)
810	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
811	}
812
813	/* Restore interrupts. */
814	ASMSetFlags(fEFlags);
815	return rc;
816	}
817
818
819	/**
820	* Exits VMX root mode operation on the current CPU.
821	*
822	* @returns VBox status code.
823	*/
824	static int hmR0VmxLeaveRootMode(void)
825	{
826	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
827
828	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
829	RTCCUINTREG fEFlags = ASMIntDisableFlags();
830
831	/* If we're for some reason not in VMX root mode, then don't leave it. */
832	RTCCUINTREG uHostCR4 = ASMGetCR4();
833
834	int rc;
835	if (uHostCR4 & X86_CR4_VMXE)
836	{
837	/* Exit VMX root mode and clear the VMX bit in CR4. */
838	VMXDisable();
839	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
840	rc = VINF_SUCCESS;
841	}
842	else
843	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
844
845	/* Restore interrupts. */
846	ASMSetFlags(fEFlags);
847	return rc;
848	}
849
850
851	/**
852	* Allocates and maps one physically contiguous page. The allocated page is
853	* zero'd out. (Used by various VT-x structures).
854	*
855	* @returns IPRT status code.
856	* @param pMemObj Pointer to the ring-0 memory object.
857	* @param ppVirt Where to store the virtual address of the
858	* allocation.
859	* @param pHCPhys Where to store the physical address of the
860	* allocation.
861	*/
862	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
863	{
864	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
865	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
866	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
867
868	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
869	if (RT_FAILURE(rc))
870	return rc;
871	ppVirt = RTR0MemObjAddress(pMemObj);
872	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
873	ASMMemZero32(*ppVirt, PAGE_SIZE);
874	return VINF_SUCCESS;
875	}
876
877
878	/**
879	* Frees and unmaps an allocated physical page.
880	*
881	* @param pMemObj Pointer to the ring-0 memory object.
882	* @param ppVirt Where to re-initialize the virtual address of
883	* allocation as 0.
884	* @param pHCPhys Where to re-initialize the physical address of the
885	* allocation as 0.
886	*/
887	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
888	{
889	AssertPtr(pMemObj);
890	AssertPtr(ppVirt);
891	AssertPtr(pHCPhys);
892	if (*pMemObj != NIL_RTR0MEMOBJ)
893	{
894	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
895	AssertRC(rc);
896	*pMemObj = NIL_RTR0MEMOBJ;
897	*ppVirt = 0;
898	*pHCPhys = 0;
899	}
900	}
901
902
903	/**
904	* Worker function to free VT-x related structures.
905	*
906	* @returns IPRT status code.
907	* @param pVM The cross context VM structure.
908	*/
909	static void hmR0VmxStructsFree(PVM pVM)
910	{
911	for (VMCPUID i = 0; i < pVM->cCpus; i++)
912	{
913	PVMCPU pVCpu = &pVM->aCpus[i];
914	AssertPtr(pVCpu);
915
916	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
917	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
918
919	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
920	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
921
922	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
923	}
924
925	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
926	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
927	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
928	#endif
929	}
930
931
932	/**
933	* Worker function to allocate VT-x related VM structures.
934	*
935	* @returns IPRT status code.
936	* @param pVM The cross context VM structure.
937	*/
938	static int hmR0VmxStructsAlloc(PVM pVM)
939	{
940	/*
941	* Initialize members up-front so we can cleanup properly on allocation failure.
942	*/
943	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
944	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946	pVM->hm.s.vmx.HCPhys##a_Name = 0;
947
948	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
949	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
952
953	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
954	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
955	#endif
956	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
957
958	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
959	for (VMCPUID i = 0; i < pVM->cCpus; i++)
960	{
961	PVMCPU pVCpu = &pVM->aCpus[i];
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
965	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
966	}
967	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
968	#undef VMXLOCAL_INIT_VM_MEMOBJ
969
970	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
971	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
972	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
973	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
974
975	/*
976	* Allocate all the VT-x structures.
977	*/
978	int rc = VINF_SUCCESS;
979	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
980	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
981	if (RT_FAILURE(rc))
982	goto cleanup;
983	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
984	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
985	#endif
986
987	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
988	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
989	{
990	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
991	&pVM->hm.s.vmx.HCPhysApicAccess);
992	if (RT_FAILURE(rc))
993	goto cleanup;
994	}
995
996	/*
997	* Initialize per-VCPU VT-x structures.
998	*/
999	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1000	{
1001	PVMCPU pVCpu = &pVM->aCpus[i];
1002	AssertPtr(pVCpu);
1003
1004	/* Allocate the VM control structure (VMCS). */
1005	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1006	if (RT_FAILURE(rc))
1007	goto cleanup;
1008
1009	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1010	if ( PDMHasApic(pVM)
1011	&& (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1012	{
1013	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1014	NULL /* pR3Ptr /, NULL / pRCPtr */);
1015	if (RT_FAILURE(rc))
1016	goto cleanup;
1017	}
1018
1019	/*
1020	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1021	* transparent accesses of specific MSRs.
1022	*
1023	* If the condition for enabling MSR bitmaps changes here, don't forget to
1024	* update HMAreMsrBitmapsAvailable().
1025	*/
1026	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1027	{
1028	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1029	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1030	if (RT_FAILURE(rc))
1031	goto cleanup;
1032	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1033	}
1034
1035	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1036	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1037	if (RT_FAILURE(rc))
1038	goto cleanup;
1039
1040	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1041	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1042	if (RT_FAILURE(rc))
1043	goto cleanup;
1044	}
1045
1046	return VINF_SUCCESS;
1047
1048	cleanup:
1049	hmR0VmxStructsFree(pVM);
1050	return rc;
1051	}
1052
1053
1054	/**
1055	* Does global VT-x initialization (called during module initialization).
1056	*
1057	* @returns VBox status code.
1058	*/
1059	VMMR0DECL(int) VMXR0GlobalInit(void)
1060	{
1061	#ifdef HMVMX_USE_FUNCTION_TABLE
1062	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1063	# ifdef VBOX_STRICT
1064	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1065	Assert(g_apfnVMExitHandlers[i]);
1066	# endif
1067	#endif
1068	return VINF_SUCCESS;
1069	}
1070
1071
1072	/**
1073	* Does global VT-x termination (called during module termination).
1074	*/
1075	VMMR0DECL(void) VMXR0GlobalTerm()
1076	{
1077	/* Nothing to do currently. */
1078	}
1079
1080
1081	/**
1082	* Sets up and activates VT-x on the current CPU.
1083	*
1084	* @returns VBox status code.
1085	* @param pCpu Pointer to the global CPU info struct.
1086	* @param pVM The cross context VM structure. Can be
1087	* NULL after a host resume operation.
1088	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1089	* fEnabledByHost is @c true).
1090	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1091	* @a fEnabledByHost is @c true).
1092	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1093	* enable VT-x on the host.
1094	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1095	*/
1096	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1097	void *pvMsrs)
1098	{
1099	Assert(pCpu);
1100	Assert(pvMsrs);
1101	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1102
1103	/* Enable VT-x if it's not already enabled by the host. */
1104	if (!fEnabledByHost)
1105	{
1106	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1107	if (RT_FAILURE(rc))
1108	return rc;
1109	}
1110
1111	/*
1112	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1113	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1114	*/
1115	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117	{
1118	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1119	pCpu->fFlushAsidBeforeUse = false;
1120	}
1121	else
1122	pCpu->fFlushAsidBeforeUse = true;
1123
1124	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125	++pCpu->cTlbFlushes;
1126
1127	return VINF_SUCCESS;
1128	}
1129
1130
1131	/**
1132	* Deactivates VT-x on the current CPU.
1133	*
1134	* @returns VBox status code.
1135	* @param pCpu Pointer to the global CPU info struct.
1136	* @param pvCpuPage Pointer to the VMXON region.
1137	* @param HCPhysCpuPage Physical address of the VMXON region.
1138	*
1139	* @remarks This function should never be called when SUPR0EnableVTx() or
1140	* similar was used to enable VT-x on the host.
1141	*/
1142	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143	{
1144	NOREF(pCpu);
1145	NOREF(pvCpuPage);
1146	NOREF(HCPhysCpuPage);
1147
1148	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1149	return hmR0VmxLeaveRootMode();
1150	}
1151
1152
1153	/**
1154	* Sets the permission bits for the specified MSR in the MSR bitmap.
1155	*
1156	* @param pVCpu The cross context virtual CPU structure.
1157	* @param uMsr The MSR value.
1158	* @param enmRead Whether reading this MSR causes a VM-exit.
1159	* @param enmWrite Whether writing this MSR causes a VM-exit.
1160	*/
1161	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1162	{
1163	int32_t iBit;
1164	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1165
1166	/*
1167	* Layout:
1168	* 0x000 - 0x3ff - Low MSR read bits
1169	* 0x400 - 0x7ff - High MSR read bits
1170	* 0x800 - 0xbff - Low MSR write bits
1171	* 0xc00 - 0xfff - High MSR write bits
1172	*/
1173	if (uMsr <= 0x00001FFF)
1174	iBit = uMsr;
1175	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1176	{
1177	iBit = uMsr - UINT32_C(0xC0000000);
1178	pbMsrBitmap += 0x400;
1179	}
1180	else
1181	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182
1183	Assert(iBit <= 0x1fff);
1184	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1185	ASMBitSet(pbMsrBitmap, iBit);
1186	else
1187	ASMBitClear(pbMsrBitmap, iBit);
1188
1189	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1190	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1191	else
1192	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1193	}
1194
1195
1196	#ifdef VBOX_STRICT
1197	/**
1198	* Gets the permission bits for the specified MSR in the MSR bitmap.
1199	*
1200	* @returns VBox status code.
1201	* @retval VINF_SUCCESS if the specified MSR is found.
1202	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1203	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1204	*
1205	* @param pVCpu The cross context virtual CPU structure.
1206	* @param uMsr The MSR.
1207	* @param penmRead Where to store the read permissions.
1208	* @param penmWrite Where to store the write permissions.
1209	*/
1210	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1211	{
1212	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1213	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1214	int32_t iBit;
1215	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1216
1217	/* See hmR0VmxSetMsrPermission() for the layout. */
1218	if (uMsr <= 0x00001FFF)
1219	iBit = uMsr;
1220	else if ( uMsr >= 0xC0000000
1221	&& uMsr <= 0xC0001FFF)
1222	{
1223	iBit = (uMsr - 0xC0000000);
1224	pbMsrBitmap += 0x400;
1225	}
1226	else
1227	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1228
1229	Assert(iBit <= 0x1fff);
1230	if (ASMBitTest(pbMsrBitmap, iBit))
1231	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1232	else
1233	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1234
1235	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1236	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1237	else
1238	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1239	return VINF_SUCCESS;
1240	}
1241	#endif /* VBOX_STRICT */
1242
1243
1244	/**
1245	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1246	* area.
1247	*
1248	* @returns VBox status code.
1249	* @param pVCpu The cross context virtual CPU structure.
1250	* @param cMsrs The number of MSRs.
1251	*/
1252	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1253	{
1254	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1255	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1256	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1257	{
1258	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1259	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1260	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1261	}
1262
1263	/* Update number of guest MSRs to load/store across the world-switch. */
1264	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1265	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1266
1267	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1268	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1269	AssertRCReturn(rc, rc);
1270
1271	/* Update the VCPU's copy of the MSR count. */
1272	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1273
1274	return VINF_SUCCESS;
1275	}
1276
1277
1278	/**
1279	* Adds a new (or updates the value of an existing) guest/host MSR
1280	* pair to be swapped during the world-switch as part of the
1281	* auto-load/store MSR area in the VMCS.
1282	*
1283	* @returns VBox status code.
1284	* @param pVCpu The cross context virtual CPU structure.
1285	* @param uMsr The MSR.
1286	* @param uGuestMsrValue Value of the guest MSR.
1287	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1288	* necessary.
1289	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1290	* its value was updated. Optional, can be NULL.
1291	*/
1292	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1293	bool *pfAddedAndUpdated)
1294	{
1295	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1296	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1297	uint32_t i;
1298	for (i = 0; i < cMsrs; i++)
1299	{
1300	if (pGuestMsr->u32Msr == uMsr)
1301	break;
1302	pGuestMsr++;
1303	}
1304
1305	bool fAdded = false;
1306	if (i == cMsrs)
1307	{
1308	++cMsrs;
1309	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1310	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1311
1312	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1313	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1314	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1315
1316	fAdded = true;
1317	}
1318
1319	/* Update the MSR values in the auto-load/store MSR area. */
1320	pGuestMsr->u32Msr = uMsr;
1321	pGuestMsr->u64Value = uGuestMsrValue;
1322
1323	/* Create/update the MSR slot in the host MSR area. */
1324	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1325	pHostMsr += i;
1326	pHostMsr->u32Msr = uMsr;
1327
1328	/*
1329	* Update the host MSR only when requested by the caller AND when we're
1330	* adding it to the auto-load/store area. Otherwise, it would have been
1331	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1332	*/
1333	bool fUpdatedMsrValue = false;
1334	if ( fAdded
1335	&& fUpdateHostMsr)
1336	{
1337	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1338	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1339	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1340	fUpdatedMsrValue = true;
1341	}
1342
1343	if (pfAddedAndUpdated)
1344	*pfAddedAndUpdated = fUpdatedMsrValue;
1345	return VINF_SUCCESS;
1346	}
1347
1348
1349	/**
1350	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1351	* auto-load/store MSR area in the VMCS.
1352	*
1353	* @returns VBox status code.
1354	* @param pVCpu The cross context virtual CPU structure.
1355	* @param uMsr The MSR.
1356	*/
1357	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1358	{
1359	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1360	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1361	for (uint32_t i = 0; i < cMsrs; i++)
1362	{
1363	/* Find the MSR. */
1364	if (pGuestMsr->u32Msr == uMsr)
1365	{
1366	/* If it's the last MSR, simply reduce the count. */
1367	if (i == cMsrs - 1)
1368	{
1369	--cMsrs;
1370	break;
1371	}
1372
1373	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1374	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1375	pLastGuestMsr += cMsrs - 1;
1376	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1377	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1378
1379	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1381	pLastHostMsr += cMsrs - 1;
1382	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1383	pHostMsr->u64Value = pLastHostMsr->u64Value;
1384	--cMsrs;
1385	break;
1386	}
1387	pGuestMsr++;
1388	}
1389
1390	/* Update the VMCS if the count changed (meaning the MSR was found). */
1391	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1392	{
1393	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1394	AssertRCReturn(rc, rc);
1395
1396	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1397	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1398	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1399
1400	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1401	return VINF_SUCCESS;
1402	}
1403
1404	return VERR_NOT_FOUND;
1405	}
1406
1407
1408	/**
1409	* Checks if the specified guest MSR is part of the auto-load/store area in
1410	* the VMCS.
1411	*
1412	* @returns true if found, false otherwise.
1413	* @param pVCpu The cross context virtual CPU structure.
1414	* @param uMsr The MSR to find.
1415	*/
1416	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1417	{
1418	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1419	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1420
1421	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1422	{
1423	if (pGuestMsr->u32Msr == uMsr)
1424	return true;
1425	}
1426	return false;
1427	}
1428
1429
1430	/**
1431	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1432	*
1433	* @param pVCpu The cross context virtual CPU structure.
1434	*
1435	* @remarks No-long-jump zone!!!
1436	*/
1437	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1438	{
1439	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1441	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1445	{
1446	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1447
1448	/*
1449	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1450	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1451	*/
1452	if (pHostMsr->u32Msr == MSR_K6_EFER)
1453	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1454	else
1455	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1456	}
1457
1458	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1459	}
1460
1461
1462	/**
1463	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1464	* perform lazy restoration of the host MSRs while leaving VT-x.
1465	*
1466	* @param pVCpu The cross context virtual CPU structure.
1467	*
1468	* @remarks No-long-jump zone!!!
1469	*/
1470	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1471	{
1472	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1473
1474	/*
1475	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1476	*/
1477	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1478	{
1479	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1480	#if HC_ARCH_BITS == 64
1481	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1482	{
1483	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1484	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1485	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1486	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1487	}
1488	#endif
1489	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1490	}
1491	}
1492
1493
1494	/**
1495	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1496	* lazily while leaving VT-x.
1497	*
1498	* @returns true if it does, false otherwise.
1499	* @param pVCpu The cross context virtual CPU structure.
1500	* @param uMsr The MSR to check.
1501	*/
1502	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1503	{
1504	NOREF(pVCpu);
1505	#if HC_ARCH_BITS == 64
1506	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1507	{
1508	switch (uMsr)
1509	{
1510	case MSR_K8_LSTAR:
1511	case MSR_K6_STAR:
1512	case MSR_K8_SF_MASK:
1513	case MSR_K8_KERNEL_GS_BASE:
1514	return true;
1515	}
1516	}
1517	#else
1518	RT_NOREF(pVCpu, uMsr);
1519	#endif
1520	return false;
1521	}
1522
1523
1524	/**
1525	* Saves a set of guest MSRs back into the guest-CPU context.
1526	*
1527	* @param pVCpu The cross context virtual CPU structure.
1528	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1529	* out-of-sync. Make sure to update the required fields
1530	* before using them.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	*/
1534	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1535	{
1536	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1540	{
1541	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1542	#if HC_ARCH_BITS == 64
1543	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1544	{
1545	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1546	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1547	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1548	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1549	}
1550	#else
1551	NOREF(pMixedCtx);
1552	#endif
1553	}
1554	}
1555
1556
1557	/**
1558	* Loads a set of guests MSRs to allow read/passthru to the guest.
1559	*
1560	* The name of this function is slightly confusing. This function does NOT
1561	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1562	* common prefix for functions dealing with "lazy restoration" of the shared
1563	* MSRs.
1564	*
1565	* @param pVCpu The cross context virtual CPU structure.
1566	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1567	* out-of-sync. Make sure to update the required fields
1568	* before using them.
1569	*
1570	* @remarks No-long-jump zone!!!
1571	*/
1572	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1573	{
1574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1575	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1576
1577	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1578	#if HC_ARCH_BITS == 64
1579	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1580	{
1581	/*
1582	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1583	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1584	* we can skip a few MSR writes.
1585	*
1586	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1587	* guest MSR values in the guest-CPU context might be different to what's currently
1588	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1589	* CPU, see @bugref{8728}.
1590	*/
1591	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1592	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1593	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1594	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1595	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1596	{
1597	#ifdef VBOX_STRICT
1598	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1599	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1600	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1601	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1602	#endif
1603	}
1604	else
1605	{
1606	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1607	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1608	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1609	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1610	}
1611	}
1612	#else
1613	RT_NOREF(pMixedCtx);
1614	#endif
1615	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1616	}
1617
1618
1619	/**
1620	* Performs lazy restoration of the set of host MSRs if they were previously
1621	* loaded with guest MSR values.
1622	*
1623	* @param pVCpu The cross context virtual CPU structure.
1624	*
1625	* @remarks No-long-jump zone!!!
1626	* @remarks The guest MSRs should have been saved back into the guest-CPU
1627	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1628	*/
1629	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1630	{
1631	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1632	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1633
1634	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1635	{
1636	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1637	#if HC_ARCH_BITS == 64
1638	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1639	{
1640	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1641	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1642	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1643	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1644	}
1645	#endif
1646	}
1647	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1648	}
1649
1650
1651	/**
1652	* Verifies that our cached values of the VMCS controls are all
1653	* consistent with what's actually present in the VMCS.
1654	*
1655	* @returns VBox status code.
1656	* @param pVCpu The cross context virtual CPU structure.
1657	*/
1658	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1659	{
1660	uint32_t u32Val;
1661	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1662	AssertRCReturn(rc, rc);
1663	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1664	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1665
1666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1667	AssertRCReturn(rc, rc);
1668	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1669	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1670
1671	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1672	AssertRCReturn(rc, rc);
1673	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1674	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1675
1676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1677	AssertRCReturn(rc, rc);
1678	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1679	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1680
1681	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1682	{
1683	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1684	AssertRCReturn(rc, rc);
1685	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1686	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1687	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1688	}
1689
1690	return VINF_SUCCESS;
1691	}
1692
1693
1694	#ifdef VBOX_STRICT
1695	/**
1696	* Verifies that our cached host EFER value has not changed
1697	* since we cached it.
1698	*
1699	* @param pVCpu The cross context virtual CPU structure.
1700	*/
1701	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1702	{
1703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1704
1705	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1706	{
1707	uint64_t u64Val;
1708	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1709	AssertRC(rc);
1710
1711	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1712	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1713	}
1714	}
1715
1716
1717	/**
1718	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1719	* VMCS are correct.
1720	*
1721	* @param pVCpu The cross context virtual CPU structure.
1722	*/
1723	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1724	{
1725	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1726
1727	/* Verify MSR counts in the VMCS are what we think it should be. */
1728	uint32_t cMsrs;
1729	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1730	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1731
1732	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1733	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1734
1735	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1736	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1737
1738	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1739	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1740	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1741	{
1742	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1743	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1744	pGuestMsr->u32Msr, cMsrs));
1745
1746	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1747	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1748	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1749
1750	/* Verify that the permissions are as expected in the MSR bitmap. */
1751	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1752	{
1753	VMXMSREXITREAD enmRead;
1754	VMXMSREXITWRITE enmWrite;
1755	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1756	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1757	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1758	{
1759	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1760	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1761	}
1762	else
1763	{
1764	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1765	pGuestMsr->u32Msr, cMsrs));
1766	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1767	pGuestMsr->u32Msr, cMsrs));
1768	}
1769	}
1770	}
1771	}
1772	#endif /* VBOX_STRICT */
1773
1774
1775	/**
1776	* Flushes the TLB using EPT.
1777	*
1778	* @returns VBox status code.
1779	* @param pVCpu The cross context virtual CPU structure of the calling
1780	* EMT. Can be NULL depending on @a enmFlush.
1781	* @param enmFlush Type of flush.
1782	*
1783	* @remarks Caller is responsible for making sure this function is called only
1784	* when NestedPaging is supported and providing @a enmFlush that is
1785	* supported by the CPU.
1786	* @remarks Can be called with interrupts disabled.
1787	*/
1788	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1789	{
1790	uint64_t au64Descriptor[2];
1791	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1792	au64Descriptor[0] = 0;
1793	else
1794	{
1795	Assert(pVCpu);
1796	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1797	}
1798	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1799
1800	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1801	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1802	rc));
1803	if ( RT_SUCCESS(rc)
1804	&& pVCpu)
1805	{
1806	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1807	}
1808	}
1809
1810
1811	/**
1812	* Flushes the TLB using VPID.
1813	*
1814	* @returns VBox status code.
1815	* @param pVM The cross context VM structure.
1816	* @param pVCpu The cross context virtual CPU structure of the calling
1817	* EMT. Can be NULL depending on @a enmFlush.
1818	* @param enmFlush Type of flush.
1819	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1820	* on @a enmFlush).
1821	*
1822	* @remarks Can be called with interrupts disabled.
1823	*/
1824	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1825	{
1826	NOREF(pVM);
1827	AssertPtr(pVM);
1828	Assert(pVM->hm.s.vmx.fVpid);
1829
1830	uint64_t au64Descriptor[2];
1831	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1832	{
1833	au64Descriptor[0] = 0;
1834	au64Descriptor[1] = 0;
1835	}
1836	else
1837	{
1838	AssertPtr(pVCpu);
1839	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1840	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1841	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1842	au64Descriptor[1] = GCPtr;
1843	}
1844
1845	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1846	AssertMsg(rc == VINF_SUCCESS,
1847	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1848	if ( RT_SUCCESS(rc)
1849	&& pVCpu)
1850	{
1851	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1852	}
1853	}
1854
1855
1856	/**
1857	* Invalidates a guest page by guest virtual address. Only relevant for
1858	* EPT/VPID, otherwise there is nothing really to invalidate.
1859	*
1860	* @returns VBox status code.
1861	* @param pVM The cross context VM structure.
1862	* @param pVCpu The cross context virtual CPU structure.
1863	* @param GCVirt Guest virtual address of the page to invalidate.
1864	*/
1865	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1866	{
1867	AssertPtr(pVM);
1868	AssertPtr(pVCpu);
1869	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1870
1871	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1872	if (!fFlushPending)
1873	{
1874	/*
1875	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1876	* See @bugref{6043} and @bugref{6177}.
1877	*
1878	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1879	* function maybe called in a loop with individual addresses.
1880	*/
1881	if (pVM->hm.s.vmx.fVpid)
1882	{
1883	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1884	{
1885	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1886	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1887	}
1888	else
1889	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1890	}
1891	else if (pVM->hm.s.fNestedPaging)
1892	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1893	}
1894
1895	return VINF_SUCCESS;
1896	}
1897
1898
1899	/**
1900	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1901	* otherwise there is nothing really to invalidate.
1902	*
1903	* @returns VBox status code.
1904	* @param pVM The cross context VM structure.
1905	* @param pVCpu The cross context virtual CPU structure.
1906	* @param GCPhys Guest physical address of the page to invalidate.
1907	*/
1908	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1909	{
1910	NOREF(pVM); NOREF(GCPhys);
1911	LogFlowFunc(("%RGp\n", GCPhys));
1912
1913	/*
1914	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1915	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1916	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1917	*/
1918	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1919	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1920	return VINF_SUCCESS;
1921	}
1922
1923
1924	/**
1925	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1926	* case where neither EPT nor VPID is supported by the CPU.
1927	*
1928	* @param pVM The cross context VM structure.
1929	* @param pVCpu The cross context virtual CPU structure.
1930	* @param pCpu Pointer to the global HM struct.
1931	*
1932	* @remarks Called with interrupts disabled.
1933	*/
1934	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1935	{
1936	AssertPtr(pVCpu);
1937	AssertPtr(pCpu);
1938	NOREF(pVM);
1939
1940	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1941
1942	Assert(pCpu->idCpu != NIL_RTCPUID);
1943	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1944	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1945	pVCpu->hm.s.fForceTLBFlush = false;
1946	return;
1947	}
1948
1949
1950	/**
1951	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1952	*
1953	* @param pVM The cross context VM structure.
1954	* @param pVCpu The cross context virtual CPU structure.
1955	* @param pCpu Pointer to the global HM CPU struct.
1956	* @remarks All references to "ASID" in this function pertains to "VPID" in
1957	* Intel's nomenclature. The reason is, to avoid confusion in compare
1958	* statements since the host-CPU copies are named "ASID".
1959	*
1960	* @remarks Called with interrupts disabled.
1961	*/
1962	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1963	{
1964	#ifdef VBOX_WITH_STATISTICS
1965	bool fTlbFlushed = false;
1966	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1967	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1968	if (!fTlbFlushed) \
1969	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1970	} while (0)
1971	#else
1972	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1973	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1974	#endif
1975
1976	AssertPtr(pVM);
1977	AssertPtr(pCpu);
1978	AssertPtr(pVCpu);
1979	Assert(pCpu->idCpu != NIL_RTCPUID);
1980
1981	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1982	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1983	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1984
1985	/*
1986	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1987	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1988	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1989	*/
1990	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1991	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1992	{
1993	++pCpu->uCurrentAsid;
1994	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1995	{
1996	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1997	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1998	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1999	}
2000
2001	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2002	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2003	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2004
2005	/*
2006	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2007	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2008	*/
2009	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2010	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2011	HMVMX_SET_TAGGED_TLB_FLUSHED();
2012	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2013	}
2014
2015	/* Check for explicit TLB flushes. */
2016	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2017	{
2018	/*
2019	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2020	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2021	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2022	* but not guest-physical mappings.
2023	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2024	*/
2025	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2026	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2027	HMVMX_SET_TAGGED_TLB_FLUSHED();
2028	}
2029
2030	pVCpu->hm.s.fForceTLBFlush = false;
2031	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2032
2033	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2034	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2035	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2036	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2037	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2038	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2039	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2040	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2041	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2042
2043	/* Update VMCS with the VPID. */
2044	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2045	AssertRC(rc);
2046
2047	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2048	}
2049
2050
2051	/**
2052	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2053	*
2054	* @returns VBox status code.
2055	* @param pVM The cross context VM structure.
2056	* @param pVCpu The cross context virtual CPU structure.
2057	* @param pCpu Pointer to the global HM CPU struct.
2058	*
2059	* @remarks Called with interrupts disabled.
2060	*/
2061	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2062	{
2063	AssertPtr(pVM);
2064	AssertPtr(pVCpu);
2065	AssertPtr(pCpu);
2066	Assert(pCpu->idCpu != NIL_RTCPUID);
2067	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2068	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2069
2070	/*
2071	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2072	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2073	*/
2074	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2075	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2076	{
2077	pVCpu->hm.s.fForceTLBFlush = true;
2078	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2079	}
2080
2081	/* Check for explicit TLB flushes. */
2082	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2083	{
2084	pVCpu->hm.s.fForceTLBFlush = true;
2085	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2086	}
2087
2088	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2089	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2090
2091	if (pVCpu->hm.s.fForceTLBFlush)
2092	{
2093	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2094	pVCpu->hm.s.fForceTLBFlush = false;
2095	}
2096	}
2097
2098
2099	/**
2100	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2101	*
2102	* @returns VBox status code.
2103	* @param pVM The cross context VM structure.
2104	* @param pVCpu The cross context virtual CPU structure.
2105	* @param pCpu Pointer to the global HM CPU struct.
2106	*
2107	* @remarks Called with interrupts disabled.
2108	*/
2109	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2110	{
2111	AssertPtr(pVM);
2112	AssertPtr(pVCpu);
2113	AssertPtr(pCpu);
2114	Assert(pCpu->idCpu != NIL_RTCPUID);
2115	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2116	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2117
2118	/*
2119	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2120	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2121	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2122	*/
2123	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2124	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2125	{
2126	pVCpu->hm.s.fForceTLBFlush = true;
2127	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2128	}
2129
2130	/* Check for explicit TLB flushes. */
2131	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2132	{
2133	/*
2134	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2135	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2136	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2137	*/
2138	pVCpu->hm.s.fForceTLBFlush = true;
2139	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2140	}
2141
2142	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2143	if (pVCpu->hm.s.fForceTLBFlush)
2144	{
2145	++pCpu->uCurrentAsid;
2146	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2147	{
2148	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2149	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2150	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2151	}
2152
2153	pVCpu->hm.s.fForceTLBFlush = false;
2154	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2155	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2156	if (pCpu->fFlushAsidBeforeUse)
2157	{
2158	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2159	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2160	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2161	{
2162	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2163	pCpu->fFlushAsidBeforeUse = false;
2164	}
2165	else
2166	{
2167	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2168	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2169	}
2170	}
2171	}
2172
2173	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2174	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2175	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2176	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2177	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2178	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2179	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2180
2181	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2182	AssertRC(rc);
2183	}
2184
2185
2186	/**
2187	* Flushes the guest TLB entry based on CPU capabilities.
2188	*
2189	* @param pVCpu The cross context virtual CPU structure.
2190	* @param pCpu Pointer to the global HM CPU struct.
2191	*/
2192	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2193	{
2194	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2195	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2196	#endif
2197	PVM pVM = pVCpu->CTX_SUFF(pVM);
2198	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2199	{
2200	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2201	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2202	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2203	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2204	default:
2205	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2206	break;
2207	}
2208
2209	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2210	}
2211
2212
2213	/**
2214	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2215	* TLB entries from the host TLB before VM-entry.
2216	*
2217	* @returns VBox status code.
2218	* @param pVM The cross context VM structure.
2219	*/
2220	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2221	{
2222	/*
2223	* Determine optimal flush type for Nested Paging.
2224	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2225	* guest execution (see hmR3InitFinalizeR0()).
2226	*/
2227	if (pVM->hm.s.fNestedPaging)
2228	{
2229	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2230	{
2231	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2232	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2233	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2234	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2235	else
2236	{
2237	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2238	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2239	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2240	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2241	}
2242
2243	/* Make sure the write-back cacheable memory type for EPT is supported. */
2244	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2245	{
2246	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2247	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2248	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2249	}
2250
2251	/* EPT requires a page-walk length of 4. */
2252	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2253	{
2254	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2255	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2256	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2257	}
2258	}
2259	else
2260	{
2261	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2262	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2263	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2264	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2265	}
2266	}
2267
2268	/*
2269	* Determine optimal flush type for VPID.
2270	*/
2271	if (pVM->hm.s.vmx.fVpid)
2272	{
2273	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2274	{
2275	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2276	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2277	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2278	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2279	else
2280	{
2281	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2282	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2283	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2284	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2285	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2286	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2287	pVM->hm.s.vmx.fVpid = false;
2288	}
2289	}
2290	else
2291	{
2292	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2293	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2294	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2295	pVM->hm.s.vmx.fVpid = false;
2296	}
2297	}
2298
2299	/*
2300	* Setup the handler for flushing tagged-TLBs.
2301	*/
2302	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2303	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2304	else if (pVM->hm.s.fNestedPaging)
2305	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2306	else if (pVM->hm.s.vmx.fVpid)
2307	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2308	else
2309	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2310	return VINF_SUCCESS;
2311	}
2312
2313
2314	/**
2315	* Sets up pin-based VM-execution controls in the VMCS.
2316	*
2317	* @returns VBox status code.
2318	* @param pVM The cross context VM structure.
2319	* @param pVCpu The cross context virtual CPU structure.
2320	*/
2321	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2322	{
2323	AssertPtr(pVM);
2324	AssertPtr(pVCpu);
2325
2326	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2327	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2328
2329	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2330	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2331
2332	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2333	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2334
2335	/* Enable the VMX preemption timer. */
2336	if (pVM->hm.s.vmx.fUsePreemptTimer)
2337	{
2338	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2339	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2340	}
2341
2342	#if 0
2343	/* Enable posted-interrupt processing. */
2344	if (pVM->hm.s.fPostedIntrs)
2345	{
2346	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2347	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2348	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2349	}
2350	#endif
2351
2352	if ((val & zap) != val)
2353	{
2354	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2355	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2356	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2357	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2358	}
2359
2360	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2361	AssertRCReturn(rc, rc);
2362
2363	pVCpu->hm.s.vmx.u32PinCtls = val;
2364	return rc;
2365	}
2366
2367
2368	/**
2369	* Sets up processor-based VM-execution controls in the VMCS.
2370	*
2371	* @returns VBox status code.
2372	* @param pVM The cross context VM structure.
2373	* @param pVCpu The cross context virtual CPU structure.
2374	*/
2375	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2376	{
2377	AssertPtr(pVM);
2378	AssertPtr(pVCpu);
2379
2380	int rc = VERR_INTERNAL_ERROR_5;
2381	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2382	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2383
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2385	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2386	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2387	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2388	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2389	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2390	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2391
2392	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2393	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2394	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2395	{
2396	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2397	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2398	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2399	}
2400
2401	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2402	if (!pVM->hm.s.fNestedPaging)
2403	{
2404	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2405	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2406	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2407	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2408	}
2409
2410	/* Use TPR shadowing if supported by the CPU. */
2411	if ( PDMHasApic(pVM)
2412	&& pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2413	{
2414	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2415	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2416	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2417	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2418	AssertRCReturn(rc, rc);
2419
2420	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2421	/* CR8 writes cause a VM-exit based on TPR threshold. */
2422	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2423	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2424	}
2425	else
2426	{
2427	/*
2428	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2429	* Set this control only for 64-bit guests.
2430	*/
2431	if (pVM->hm.s.fAllow64BitGuests)
2432	{
2433	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2434	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2435	}
2436	}
2437
2438	/* Use MSR-bitmaps if supported by the CPU. */
2439	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2440	{
2441	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2442
2443	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2444	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2445	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2446	AssertRCReturn(rc, rc);
2447
2448	/*
2449	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2450	* automatically using dedicated fields in the VMCS.
2451	*/
2452	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2455	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2456	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2457
2458	#if HC_ARCH_BITS == 64
2459	/*
2460	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2461	*/
2462	if (pVM->hm.s.fAllow64BitGuests)
2463	{
2464	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2465	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2466	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2467	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2468	}
2469	#endif
2470	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2471	}
2472
2473	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2474	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2475	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2476
2477	if ((val & zap) != val)
2478	{
2479	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2480	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2481	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2482	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2483	}
2484
2485	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2486	AssertRCReturn(rc, rc);
2487
2488	pVCpu->hm.s.vmx.u32ProcCtls = val;
2489
2490	/*
2491	* Secondary processor-based VM-execution controls.
2492	*/
2493	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2494	{
2495	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2496	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2497
2498	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2499	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2500
2501	if (pVM->hm.s.fNestedPaging)
2502	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2503	else
2504	{
2505	/*
2506	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2507	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2508	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2509	*/
2510	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2511	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2512	}
2513
2514	if (pVM->hm.s.vmx.fVpid)
2515	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2516
2517	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2518	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2519
2520	#if 0
2521	if (pVM->hm.s.fVirtApicRegs)
2522	{
2523	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2524	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2525
2526	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2527	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2528	}
2529	#endif
2530
2531	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2532	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2533	* done dynamically. */
2534	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2535	{
2536	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2537	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2538	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2539	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2540	AssertRCReturn(rc, rc);
2541	}
2542
2543	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2544	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2545
2546	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2547	&& pVM->hm.s.vmx.cPleGapTicks
2548	&& pVM->hm.s.vmx.cPleWindowTicks)
2549	{
2550	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2551
2552	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2553	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2554	AssertRCReturn(rc, rc);
2555	}
2556
2557	if ((val & zap) != val)
2558	{
2559	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2560	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2561	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2562	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2563	}
2564
2565	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2566	AssertRCReturn(rc, rc);
2567
2568	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2569	}
2570	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2571	{
2572	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2573	"available\n"));
2574	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2575	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2576	}
2577
2578	return VINF_SUCCESS;
2579	}
2580
2581
2582	/**
2583	* Sets up miscellaneous (everything other than Pin & Processor-based
2584	* VM-execution) control fields in the VMCS.
2585	*
2586	* @returns VBox status code.
2587	* @param pVM The cross context VM structure.
2588	* @param pVCpu The cross context virtual CPU structure.
2589	*/
2590	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2591	{
2592	NOREF(pVM);
2593	AssertPtr(pVM);
2594	AssertPtr(pVCpu);
2595
2596	int rc = VERR_GENERAL_FAILURE;
2597
2598	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2599	#if 0
2600	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2601	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2602	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2603
2604	/*
2605	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2606	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2607	* We thus use the exception bitmap to control it rather than use both.
2608	*/
2609	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2610	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2611
2612	/** @todo Explore possibility of using IO-bitmaps. */
2613	/* All IO & IOIO instructions cause VM-exits. */
2614	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2615	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2616
2617	/* Initialize the MSR-bitmap area. */
2618	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2619	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2620	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2621	AssertRCReturn(rc, rc);
2622	#endif
2623
2624	/* Setup MSR auto-load/store area. */
2625	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2626	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2627	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2628	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2629	AssertRCReturn(rc, rc);
2630
2631	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2632	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2633	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2634	AssertRCReturn(rc, rc);
2635
2636	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2637	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2638	AssertRCReturn(rc, rc);
2639
2640	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2641	#if 0
2642	/* Setup debug controls */
2643	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2644	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2645	AssertRCReturn(rc, rc);
2646	#endif
2647
2648	return rc;
2649	}
2650
2651
2652	/**
2653	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2654	*
2655	* We shall setup those exception intercepts that don't change during the
2656	* lifetime of the VM here. The rest are done dynamically while loading the
2657	* guest state.
2658	*
2659	* @returns VBox status code.
2660	* @param pVM The cross context VM structure.
2661	* @param pVCpu The cross context virtual CPU structure.
2662	*/
2663	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2664	{
2665	AssertPtr(pVM);
2666	AssertPtr(pVCpu);
2667
2668	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2669
2670	uint32_t u32XcptBitmap = 0;
2671
2672	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2673	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2674
2675	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2676	and writes, and because recursive #DBs can cause the CPU hang, we must always
2677	intercept #DB. */
2678	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2679
2680	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2681	if (!pVM->hm.s.fNestedPaging)
2682	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2683
2684	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2685	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2686	AssertRCReturn(rc, rc);
2687	return rc;
2688	}
2689
2690
2691	/**
2692	* Sets up the initial guest-state mask. The guest-state mask is consulted
2693	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2694	* for the nested virtualization case (as it would cause a VM-exit).
2695	*
2696	* @param pVCpu The cross context virtual CPU structure.
2697	*/
2698	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2699	{
2700	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2701	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2702	return VINF_SUCCESS;
2703	}
2704
2705
2706	/**
2707	* Does per-VM VT-x initialization.
2708	*
2709	* @returns VBox status code.
2710	* @param pVM The cross context VM structure.
2711	*/
2712	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2713	{
2714	LogFlowFunc(("pVM=%p\n", pVM));
2715
2716	int rc = hmR0VmxStructsAlloc(pVM);
2717	if (RT_FAILURE(rc))
2718	{
2719	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2720	return rc;
2721	}
2722
2723	return VINF_SUCCESS;
2724	}
2725
2726
2727	/**
2728	* Does per-VM VT-x termination.
2729	*
2730	* @returns VBox status code.
2731	* @param pVM The cross context VM structure.
2732	*/
2733	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2734	{
2735	LogFlowFunc(("pVM=%p\n", pVM));
2736
2737	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2738	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2739	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2740	#endif
2741	hmR0VmxStructsFree(pVM);
2742	return VINF_SUCCESS;
2743	}
2744
2745
2746	/**
2747	* Sets up the VM for execution under VT-x.
2748	* This function is only called once per-VM during initialization.
2749	*
2750	* @returns VBox status code.
2751	* @param pVM The cross context VM structure.
2752	*/
2753	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2754	{
2755	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2756	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2757
2758	LogFlowFunc(("pVM=%p\n", pVM));
2759
2760	/*
2761	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2762	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2763	*/
2764	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2765	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2766	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2767	{
2768	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2769	return VERR_INTERNAL_ERROR;
2770	}
2771
2772	/* Initialize these always, see hmR3InitFinalizeR0().*/
2773	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2774	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2775
2776	/* Setup the tagged-TLB flush handlers. */
2777	int rc = hmR0VmxSetupTaggedTlb(pVM);
2778	if (RT_FAILURE(rc))
2779	{
2780	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2781	return rc;
2782	}
2783
2784	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2785	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2786	#if HC_ARCH_BITS == 64
2787	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2788	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2789	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2790	{
2791	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2792	}
2793	#endif
2794
2795	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2796	RTCCUINTREG uHostCR4 = ASMGetCR4();
2797	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2798	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2799
2800	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2801	{
2802	PVMCPU pVCpu = &pVM->aCpus[i];
2803	AssertPtr(pVCpu);
2804	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2805
2806	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2807	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2808
2809	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2810	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2811	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2812
2813	/* Set revision dword at the beginning of the VMCS structure. */
2814	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2815
2816	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2817	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2818	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2819	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2820
2821	/* Load this VMCS as the current VMCS. */
2822	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2823	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2827	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2828	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2829
2830	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2831	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2832	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2833
2834	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2835	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2836	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2837
2838	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2839	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2840	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2841
2842	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2843	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2844	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2845
2846	#if HC_ARCH_BITS == 32
2847	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2848	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2849	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2850	#endif
2851
2852	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2853	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2854	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2855	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2856
2857	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2858
2859	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2860	}
2861
2862	return VINF_SUCCESS;
2863	}
2864
2865
2866	/**
2867	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2868	* the VMCS.
2869	*
2870	* @returns VBox status code.
2871	* @param pVM The cross context VM structure.
2872	* @param pVCpu The cross context virtual CPU structure.
2873	*/
2874	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2875	{
2876	NOREF(pVM); NOREF(pVCpu);
2877
2878	RTCCUINTREG uReg = ASMGetCR0();
2879	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2880	AssertRCReturn(rc, rc);
2881
2882	uReg = ASMGetCR3();
2883	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2884	AssertRCReturn(rc, rc);
2885
2886	uReg = ASMGetCR4();
2887	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2888	AssertRCReturn(rc, rc);
2889	return rc;
2890	}
2891
2892
2893	#if HC_ARCH_BITS == 64
2894	/**
2895	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2896	* requirements. See hmR0VmxSaveHostSegmentRegs().
2897	*/
2898	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2899	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2900	{ \
2901	bool fValidSelector = true; \
2902	if ((selValue) & X86_SEL_LDT) \
2903	{ \
2904	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2905	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2906	} \
2907	if (fValidSelector) \
2908	{ \
2909	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2910	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2911	} \
2912	(selValue) = 0; \
2913	}
2914	#endif
2915
2916
2917	/**
2918	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2919	* the host-state area in the VMCS.
2920	*
2921	* @returns VBox status code.
2922	* @param pVM The cross context VM structure.
2923	* @param pVCpu The cross context virtual CPU structure.
2924	*/
2925	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2926	{
2927	int rc = VERR_INTERNAL_ERROR_5;
2928
2929	#if HC_ARCH_BITS == 64
2930	/*
2931	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2932	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2933	*
2934	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2935	* Was observed booting Solaris10u10 32-bit guest.
2936	*/
2937	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2938	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2939	{
2940	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2941	pVCpu->idCpu));
2942	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2943	}
2944	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2945	#else
2946	RT_NOREF(pVCpu);
2947	#endif
2948
2949	/*
2950	* Host DS, ES, FS and GS segment registers.
2951	*/
2952	#if HC_ARCH_BITS == 64
2953	RTSEL uSelDS = ASMGetDS();
2954	RTSEL uSelES = ASMGetES();
2955	RTSEL uSelFS = ASMGetFS();
2956	RTSEL uSelGS = ASMGetGS();
2957	#else
2958	RTSEL uSelDS = 0;
2959	RTSEL uSelES = 0;
2960	RTSEL uSelFS = 0;
2961	RTSEL uSelGS = 0;
2962	#endif
2963
2964	/*
2965	* Host CS and SS segment registers.
2966	*/
2967	RTSEL uSelCS = ASMGetCS();
2968	RTSEL uSelSS = ASMGetSS();
2969
2970	/*
2971	* Host TR segment register.
2972	*/
2973	RTSEL uSelTR = ASMGetTR();
2974
2975	#if HC_ARCH_BITS == 64
2976	/*
2977	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2978	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2979	*/
2980	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2981	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2982	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2983	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2984	# undef VMXLOCAL_ADJUST_HOST_SEG
2985	#endif
2986
2987	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2988	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2989	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2990	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2991	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2992	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2993	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2994	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2995	Assert(uSelCS);
2996	Assert(uSelTR);
2997
2998	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2999	#if 0
3000	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3001	Assert(uSelSS != 0);
3002	#endif
3003
3004	/* Write these host selector fields into the host-state area in the VMCS. */
3005	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3006	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3007	#if HC_ARCH_BITS == 64
3008	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3009	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3010	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3011	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3012	#else
3013	NOREF(uSelDS);
3014	NOREF(uSelES);
3015	NOREF(uSelFS);
3016	NOREF(uSelGS);
3017	#endif
3018	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3019	AssertRCReturn(rc, rc);
3020
3021	/*
3022	* Host GDTR and IDTR.
3023	*/
3024	RTGDTR Gdtr;
3025	RTIDTR Idtr;
3026	RT_ZERO(Gdtr);
3027	RT_ZERO(Idtr);
3028	ASMGetGDTR(&Gdtr);
3029	ASMGetIDTR(&Idtr);
3030	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3031	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3032	AssertRCReturn(rc, rc);
3033
3034	#if HC_ARCH_BITS == 64
3035	/*
3036	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3037	* maximum limit (0xffff) on every VM-exit.
3038	*/
3039	if (Gdtr.cbGdt != 0xffff)
3040	{
3041	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3042	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3043	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3044	}
3045
3046	/*
3047	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3048	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3049	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3050	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3051	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3052	* hosts where we are pretty sure it won't cause trouble.
3053	*/
3054	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3055	if (Idtr.cbIdt < 0x0fff)
3056	# else
3057	if (Idtr.cbIdt != 0xffff)
3058	# endif
3059	{
3060	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3061	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3062	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3063	}
3064	#endif
3065
3066	/*
3067	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3068	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3069	*/
3070	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3071	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3072	VERR_VMX_INVALID_HOST_STATE);
3073
3074	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3075	#if HC_ARCH_BITS == 64
3076	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3077
3078	/*
3079	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3080	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3081	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3082	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3083	*
3084	* [1] See Intel spec. 3.5 "System Descriptor Types".
3085	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3086	*/
3087	Assert(pDesc->System.u4Type == 11);
3088	if ( pDesc->System.u16LimitLow != 0x67
3089	\|\| pDesc->System.u4LimitHigh)
3090	{
3091	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3092	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3093	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3094	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3095	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3096
3097	/* Store the GDTR here as we need it while restoring TR. */
3098	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3099	}
3100	#else
3101	NOREF(pVM);
3102	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3103	#endif
3104	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3105	AssertRCReturn(rc, rc);
3106
3107	/*
3108	* Host FS base and GS base.
3109	*/
3110	#if HC_ARCH_BITS == 64
3111	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3112	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3113	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3114	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3115	AssertRCReturn(rc, rc);
3116
3117	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3118	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3119	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3120	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3121	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3122	#endif
3123	return rc;
3124	}
3125
3126
3127	/**
3128	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3129	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3130	* the host after every successful VM-exit.
3131	*
3132	* @returns VBox status code.
3133	* @param pVM The cross context VM structure.
3134	* @param pVCpu The cross context virtual CPU structure.
3135	*
3136	* @remarks No-long-jump zone!!!
3137	*/
3138	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3139	{
3140	NOREF(pVM);
3141
3142	AssertPtr(pVCpu);
3143	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3144
3145	/*
3146	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3147	* rather than swapping them on every VM-entry.
3148	*/
3149	hmR0VmxLazySaveHostMsrs(pVCpu);
3150
3151	/*
3152	* Host Sysenter MSRs.
3153	*/
3154	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3155	#if HC_ARCH_BITS == 32
3156	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3157	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3158	#else
3159	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3160	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3161	#endif
3162	AssertRCReturn(rc, rc);
3163
3164	/*
3165	* Host EFER MSR.
3166	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3167	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3168	*/
3169	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3170	{
3171	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3172	AssertRCReturn(rc, rc);
3173	}
3174
3175	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3176	* hmR0VmxLoadGuestExitCtls() !! */
3177
3178	return rc;
3179	}
3180
3181
3182	/**
3183	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3184	*
3185	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3186	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3187	* hmR0VMxLoadGuestEntryCtls().
3188	*
3189	* @returns true if we need to load guest EFER, false otherwise.
3190	* @param pVCpu The cross context virtual CPU structure.
3191	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3192	* out-of-sync. Make sure to update the required fields
3193	* before using them.
3194	*
3195	* @remarks Requires EFER, CR4.
3196	* @remarks No-long-jump zone!!!
3197	*/
3198	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3199	{
3200	#ifdef HMVMX_ALWAYS_SWAP_EFER
3201	return true;
3202	#endif
3203
3204	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3205	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3206	if (CPUMIsGuestInLongMode(pVCpu))
3207	return false;
3208	#endif
3209
3210	PVM pVM = pVCpu->CTX_SUFF(pVM);
3211	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3212	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3213
3214	/*
3215	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3216	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3217	*/
3218	if ( CPUMIsGuestInLongMode(pVCpu)
3219	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3220	{
3221	return true;
3222	}
3223
3224	/*
3225	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3226	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3227	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3228	*/
3229	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3230	&& (pMixedCtx->cr0 & X86_CR0_PG)
3231	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3232	{
3233	/* Assert that host is PAE capable. */
3234	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3235	return true;
3236	}
3237
3238	/** @todo Check the latest Intel spec. for any other bits,
3239	* like SMEP/SMAP? */
3240	return false;
3241	}
3242
3243
3244	/**
3245	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3246	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3247	* controls".
3248	*
3249	* @returns VBox status code.
3250	* @param pVCpu The cross context virtual CPU structure.
3251	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3252	* out-of-sync. Make sure to update the required fields
3253	* before using them.
3254	*
3255	* @remarks Requires EFER.
3256	* @remarks No-long-jump zone!!!
3257	*/
3258	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3259	{
3260	int rc = VINF_SUCCESS;
3261	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3262	{
3263	PVM pVM = pVCpu->CTX_SUFF(pVM);
3264	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3265	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3266
3267	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3268	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3269
3270	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3271	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3272	{
3273	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3274	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3275	}
3276	else
3277	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3278
3279	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3280	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3281	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3282	{
3283	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3284	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3285	}
3286
3287	/*
3288	* The following should -not- be set (since we're not in SMM mode):
3289	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3290	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3291	*/
3292
3293	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3294	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3295
3296	if ((val & zap) != val)
3297	{
3298	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3299	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3300	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3301	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3302	}
3303
3304	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3305	AssertRCReturn(rc, rc);
3306
3307	pVCpu->hm.s.vmx.u32EntryCtls = val;
3308	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3309	}
3310	return rc;
3311	}
3312
3313
3314	/**
3315	* Sets up the VM-exit controls in the VMCS.
3316	*
3317	* @returns VBox status code.
3318	* @param pVCpu The cross context virtual CPU structure.
3319	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3320	* out-of-sync. Make sure to update the required fields
3321	* before using them.
3322	*
3323	* @remarks Requires EFER.
3324	*/
3325	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3326	{
3327	NOREF(pMixedCtx);
3328
3329	int rc = VINF_SUCCESS;
3330	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3331	{
3332	PVM pVM = pVCpu->CTX_SUFF(pVM);
3333	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3334	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3335
3336	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3337	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3338
3339	/*
3340	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3341	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3342	*/
3343	#if HC_ARCH_BITS == 64
3344	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3345	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3346	#else
3347	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3348	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3349	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3350	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3351	{
3352	/* The switcher returns to long mode, EFER is managed by the switcher. */
3353	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3354	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3355	}
3356	else
3357	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3358	#endif
3359
3360	/* If the newer VMCS fields for managing EFER exists, use it. */
3361	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3362	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3363	{
3364	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3365	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3366	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3367	}
3368
3369	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3370	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3371
3372	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3373	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3374	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3375
3376	if ( pVM->hm.s.vmx.fUsePreemptTimer
3377	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3378	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3379
3380	if ((val & zap) != val)
3381	{
3382	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3383	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3384	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3385	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3386	}
3387
3388	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3389	AssertRCReturn(rc, rc);
3390
3391	pVCpu->hm.s.vmx.u32ExitCtls = val;
3392	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3393	}
3394	return rc;
3395	}
3396
3397
3398	/**
3399	* Sets the TPR threshold in the VMCS.
3400	*
3401	* @returns VBox status code.
3402	* @param pVCpu The cross context virtual CPU structure.
3403	* @param u32TprThreshold The TPR threshold (task-priority class only).
3404	*/
3405	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3406	{
3407	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3408	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3409	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3410	}
3411
3412
3413	/**
3414	* Loads the guest APIC and related state.
3415	*
3416	* @returns VBox status code.
3417	* @param pVCpu The cross context virtual CPU structure.
3418	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3419	* out-of-sync. Make sure to update the required fields
3420	* before using them.
3421	*
3422	* @remarks No-long-jump zone!!!
3423	*/
3424	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3425	{
3426	NOREF(pMixedCtx);
3427
3428	int rc = VINF_SUCCESS;
3429	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3430	{
3431	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3432	&& APICIsEnabled(pVCpu))
3433	{
3434	/*
3435	* Setup TPR shadowing.
3436	*/
3437	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3438	{
3439	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3440
3441	bool fPendingIntr = false;
3442	uint8_t u8Tpr = 0;
3443	uint8_t u8PendingIntr = 0;
3444	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3445	AssertRCReturn(rc, rc);
3446
3447	/*
3448	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3449	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3450	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3451	*/
3452	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3453	uint32_t u32TprThreshold = 0;
3454	if (fPendingIntr)
3455	{
3456	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3457	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3458	const uint8_t u8TprPriority = u8Tpr >> 4;
3459	if (u8PendingPriority <= u8TprPriority)
3460	u32TprThreshold = u8PendingPriority;
3461	}
3462
3463	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3464	AssertRCReturn(rc, rc);
3465	}
3466	}
3467	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3468	}
3469
3470	return rc;
3471	}
3472
3473
3474	/**
3475	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3476	*
3477	* @returns Guest's interruptibility-state.
3478	* @param pVCpu The cross context virtual CPU structure.
3479	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3480	* out-of-sync. Make sure to update the required fields
3481	* before using them.
3482	*
3483	* @remarks No-long-jump zone!!!
3484	*/
3485	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3486	{
3487	/*
3488	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3489	*/
3490	uint32_t uIntrState = 0;
3491	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3492	{
3493	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3494	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3495	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3496	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3497	{
3498	if (pMixedCtx->eflags.Bits.u1IF)
3499	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3500	else
3501	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3502	}
3503	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3504	{
3505	/*
3506	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3507	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3508	*/
3509	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3510	}
3511	}
3512
3513	/*
3514	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3515	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3516	* setting this would block host-NMIs and IRET will not clear the blocking.
3517	*
3518	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3519	*/
3520	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3521	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3522	{
3523	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3524	}
3525
3526	return uIntrState;
3527	}
3528
3529
3530	/**
3531	* Loads the guest's interruptibility-state into the guest-state area in the
3532	* VMCS.
3533	*
3534	* @returns VBox status code.
3535	* @param pVCpu The cross context virtual CPU structure.
3536	* @param uIntrState The interruptibility-state to set.
3537	*/
3538	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3539	{
3540	NOREF(pVCpu);
3541	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3542	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3543	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3544	AssertRC(rc);
3545	return rc;
3546	}
3547
3548
3549	/**
3550	* Loads the exception intercepts required for guest execution in the VMCS.
3551	*
3552	* @returns VBox status code.
3553	* @param pVCpu The cross context virtual CPU structure.
3554	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3555	* out-of-sync. Make sure to update the required fields
3556	* before using them.
3557	*/
3558	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3559	{
3560	NOREF(pMixedCtx);
3561	int rc = VINF_SUCCESS;
3562	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3563	{
3564	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3565	if (pVCpu->hm.s.fGIMTrapXcptUD)
3566	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3567	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3568	else
3569	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3570	#endif
3571
3572	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3573	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3574
3575	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3576	AssertRCReturn(rc, rc);
3577
3578	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3579	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3580	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3581	}
3582	return rc;
3583	}
3584
3585
3586	/**
3587	* Loads the guest's RIP into the guest-state area in the VMCS.
3588	*
3589	* @returns VBox status code.
3590	* @param pVCpu The cross context virtual CPU structure.
3591	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3592	* out-of-sync. Make sure to update the required fields
3593	* before using them.
3594	*
3595	* @remarks No-long-jump zone!!!
3596	*/
3597	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3598	{
3599	int rc = VINF_SUCCESS;
3600	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3601	{
3602	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3603	AssertRCReturn(rc, rc);
3604
3605	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3606	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3607	HMCPU_CF_VALUE(pVCpu)));
3608	}
3609	return rc;
3610	}
3611
3612
3613	/**
3614	* Loads the guest's RSP into the guest-state area in the VMCS.
3615	*
3616	* @returns VBox status code.
3617	* @param pVCpu The cross context virtual CPU structure.
3618	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3619	* out-of-sync. Make sure to update the required fields
3620	* before using them.
3621	*
3622	* @remarks No-long-jump zone!!!
3623	*/
3624	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3625	{
3626	int rc = VINF_SUCCESS;
3627	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3628	{
3629	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3630	AssertRCReturn(rc, rc);
3631
3632	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3633	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3634	}
3635	return rc;
3636	}
3637
3638
3639	/**
3640	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3641	*
3642	* @returns VBox status code.
3643	* @param pVCpu The cross context virtual CPU structure.
3644	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3645	* out-of-sync. Make sure to update the required fields
3646	* before using them.
3647	*
3648	* @remarks No-long-jump zone!!!
3649	*/
3650	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3651	{
3652	int rc = VINF_SUCCESS;
3653	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3654	{
3655	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3656	Let us assert it as such and use 32-bit VMWRITE. */
3657	Assert(!(pMixedCtx->rflags.u64 >> 32));
3658	X86EFLAGS Eflags = pMixedCtx->eflags;
3659	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3660	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3661	* These will never be cleared/set, unless some other part of the VMM
3662	* code is buggy - in which case we're better of finding and fixing
3663	* those bugs than hiding them. */
3664	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3665	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3666	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3667	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3668
3669	/*
3670	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3671	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3672	*/
3673	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3674	{
3675	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3676	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3677	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3678	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3679	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3680	}
3681
3682	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3683	AssertRCReturn(rc, rc);
3684
3685	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3686	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3687	}
3688	return rc;
3689	}
3690
3691
3692	/**
3693	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3694	*
3695	* @returns VBox status code.
3696	* @param pVCpu The cross context virtual CPU structure.
3697	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3698	* out-of-sync. Make sure to update the required fields
3699	* before using them.
3700	*
3701	* @remarks No-long-jump zone!!!
3702	*/
3703	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3704	{
3705	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3706	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3707	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3708	AssertRCReturn(rc, rc);
3709	return rc;
3710	}
3711
3712
3713	/**
3714	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3715	* CR0 is partially shared with the host and we have to consider the FPU bits.
3716	*
3717	* @returns VBox status code.
3718	* @param pVCpu The cross context virtual CPU structure.
3719	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3720	* out-of-sync. Make sure to update the required fields
3721	* before using them.
3722	*
3723	* @remarks No-long-jump zone!!!
3724	*/
3725	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3726	{
3727	/*
3728	* Guest CR0.
3729	* Guest FPU.
3730	*/
3731	int rc = VINF_SUCCESS;
3732	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3733	{
3734	Assert(!(pMixedCtx->cr0 >> 32));
3735	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3736	PVM pVM = pVCpu->CTX_SUFF(pVM);
3737
3738	/* The guest's view (read access) of its CR0 is unblemished. */
3739	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3740	AssertRCReturn(rc, rc);
3741	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3742
3743	/* Setup VT-x's view of the guest CR0. */
3744	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3745	if (pVM->hm.s.fNestedPaging)
3746	{
3747	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3748	{
3749	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3750	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3751	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3752	}
3753	else
3754	{
3755	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3756	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3757	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3758	}
3759
3760	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3761	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3762	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3763
3764	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3765	AssertRCReturn(rc, rc);
3766	}
3767	else
3768	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3769
3770	/*
3771	* Guest FPU bits.
3772	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3773	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3774	*/
3775	u32GuestCR0 \|= X86_CR0_NE;
3776	bool fInterceptNM = false;
3777	if (CPUMIsGuestFPUStateActive(pVCpu))
3778	{
3779	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3780	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3781	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3782	}
3783	else
3784	{
3785	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3786	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3787	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3788	}
3789
3790	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3791	bool fInterceptMF = false;
3792	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3793	fInterceptMF = true;
3794
3795	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3796	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3797	{
3798	Assert(PDMVmmDevHeapIsEnabled(pVM));
3799	Assert(pVM->hm.s.vmx.pRealModeTSS);
3800	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3801	fInterceptNM = true;
3802	fInterceptMF = true;
3803	}
3804	else
3805	{
3806	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3807	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3808	}
3809	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3810
3811	if (fInterceptNM)
3812	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3813	else
3814	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3815
3816	if (fInterceptMF)
3817	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3818	else
3819	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3820
3821	/* Additional intercepts for debugging, define these yourself explicitly. */
3822	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3823	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3824	\| RT_BIT(X86_XCPT_BP)
3825	\| RT_BIT(X86_XCPT_DE)
3826	\| RT_BIT(X86_XCPT_NM)
3827	\| RT_BIT(X86_XCPT_TS)
3828	\| RT_BIT(X86_XCPT_UD)
3829	\| RT_BIT(X86_XCPT_NP)
3830	\| RT_BIT(X86_XCPT_SS)
3831	\| RT_BIT(X86_XCPT_GP)
3832	\| RT_BIT(X86_XCPT_PF)
3833	\| RT_BIT(X86_XCPT_MF)
3834	;
3835	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3836	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3837	#endif
3838
3839	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3840
3841	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3842	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3843	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3844	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3845	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3846	else
3847	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3848
3849	u32GuestCR0 \|= uSetCR0;
3850	u32GuestCR0 &= uZapCR0;
3851	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3852
3853	/* Write VT-x's view of the guest CR0 into the VMCS. */
3854	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3855	AssertRCReturn(rc, rc);
3856	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3857	uZapCR0));
3858
3859	/*
3860	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3861	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3862	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3863	*/
3864	uint32_t u32CR0Mask = 0;
3865	u32CR0Mask = X86_CR0_PE
3866	\| X86_CR0_NE
3867	\| X86_CR0_WP
3868	\| X86_CR0_PG
3869	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3870	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3871	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3872
3873	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3874	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3875	* and @bugref{6944}. */
3876	#if 0
3877	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3878	u32CR0Mask &= ~X86_CR0_PE;
3879	#endif
3880	if (pVM->hm.s.fNestedPaging)
3881	u32CR0Mask &= ~X86_CR0_WP;
3882
3883	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3884	if (fInterceptNM)
3885	{
3886	u32CR0Mask \|= X86_CR0_TS
3887	\| X86_CR0_MP;
3888	}
3889
3890	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3891	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3892	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3893	AssertRCReturn(rc, rc);
3894	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3895
3896	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3897	}
3898	return rc;
3899	}
3900
3901
3902	/**
3903	* Loads the guest control registers (CR3, CR4) into the guest-state area
3904	* in the VMCS.
3905	*
3906	* @returns VBox strict status code.
3907	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3908	* without unrestricted guest access and the VMMDev is not presently
3909	* mapped (e.g. EFI32).
3910	*
3911	* @param pVCpu The cross context virtual CPU structure.
3912	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3913	* out-of-sync. Make sure to update the required fields
3914	* before using them.
3915	*
3916	* @remarks No-long-jump zone!!!
3917	*/
3918	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3919	{
3920	int rc = VINF_SUCCESS;
3921	PVM pVM = pVCpu->CTX_SUFF(pVM);
3922
3923	/*
3924	* Guest CR2.
3925	* It's always loaded in the assembler code. Nothing to do here.
3926	*/
3927
3928	/*
3929	* Guest CR3.
3930	*/
3931	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3932	{
3933	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3934	if (pVM->hm.s.fNestedPaging)
3935	{
3936	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3937
3938	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3939	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3940	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3941	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3942
3943	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3944	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3945	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3946
3947	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3948	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3949	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3950	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3951	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3952	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3953	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3954
3955	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3956	AssertRCReturn(rc, rc);
3957	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3958
3959	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3960	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3961	{
3962	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3963	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3964	{
3965	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3966	AssertRCReturn(rc, rc);
3967	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3968	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3969	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3970	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3971	AssertRCReturn(rc, rc);
3972	}
3973
3974	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3975	have Unrestricted Execution to handle the guest when it's not using paging. */
3976	GCPhysGuestCR3 = pMixedCtx->cr3;
3977	}
3978	else
3979	{
3980	/*
3981	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3982	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3983	* EPT takes care of translating it to host-physical addresses.
3984	*/
3985	RTGCPHYS GCPhys;
3986	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3987
3988	/* We obtain it here every time as the guest could have relocated this PCI region. */
3989	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3990	if (RT_SUCCESS(rc))
3991	{ /* likely */ }
3992	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3993	{
3994	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3995	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3996	}
3997	else
3998	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3999
4000	GCPhysGuestCR3 = GCPhys;
4001	}
4002
4003	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4004	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4005	}
4006	else
4007	{
4008	/* Non-nested paging case, just use the hypervisor's CR3. */
4009	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4010
4011	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4012	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4013	}
4014	AssertRCReturn(rc, rc);
4015
4016	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4017	}
4018
4019	/*
4020	* Guest CR4.
4021	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4022	*/
4023	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4024	{
4025	Assert(!(pMixedCtx->cr4 >> 32));
4026	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4027
4028	/* The guest's view of its CR4 is unblemished. */
4029	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4030	AssertRCReturn(rc, rc);
4031	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4032
4033	/* Setup VT-x's view of the guest CR4. */
4034	/*
4035	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4036	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4037	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4038	*/
4039	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4040	{
4041	Assert(pVM->hm.s.vmx.pRealModeTSS);
4042	Assert(PDMVmmDevHeapIsEnabled(pVM));
4043	u32GuestCR4 &= ~X86_CR4_VME;
4044	}
4045
4046	if (pVM->hm.s.fNestedPaging)
4047	{
4048	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4049	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4050	{
4051	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4052	u32GuestCR4 \|= X86_CR4_PSE;
4053	/* Our identity mapping is a 32-bit page directory. */
4054	u32GuestCR4 &= ~X86_CR4_PAE;
4055	}
4056	/* else use guest CR4.*/
4057	}
4058	else
4059	{
4060	/*
4061	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4062	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4063	*/
4064	switch (pVCpu->hm.s.enmShadowMode)
4065	{
4066	case PGMMODE_REAL: /* Real-mode. */
4067	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4068	case PGMMODE_32_BIT: /* 32-bit paging. */
4069	{
4070	u32GuestCR4 &= ~X86_CR4_PAE;
4071	break;
4072	}
4073
4074	case PGMMODE_PAE: /* PAE paging. */
4075	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4076	{
4077	u32GuestCR4 \|= X86_CR4_PAE;
4078	break;
4079	}
4080
4081	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4082	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4083	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4084	break;
4085	#endif
4086	default:
4087	AssertFailed();
4088	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4089	}
4090	}
4091
4092	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4093	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4094	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4095	u32GuestCR4 \|= uSetCR4;
4096	u32GuestCR4 &= uZapCR4;
4097
4098	/* Write VT-x's view of the guest CR4 into the VMCS. */
4099	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4100	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4101	AssertRCReturn(rc, rc);
4102
4103	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4104	uint32_t u32CR4Mask = X86_CR4_VME
4105	\| X86_CR4_PAE
4106	\| X86_CR4_PGE
4107	\| X86_CR4_PSE
4108	\| X86_CR4_VMXE;
4109	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4110	u32CR4Mask \|= X86_CR4_OSXSAVE;
4111	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4112	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4113	AssertRCReturn(rc, rc);
4114
4115	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4116	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4117
4118	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4119	}
4120	return rc;
4121	}
4122
4123
4124	/**
4125	* Loads the guest debug registers into the guest-state area in the VMCS.
4126	*
4127	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4128	*
4129	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4130	*
4131	* @returns VBox status code.
4132	* @param pVCpu The cross context virtual CPU structure.
4133	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4134	* out-of-sync. Make sure to update the required fields
4135	* before using them.
4136	*
4137	* @remarks No-long-jump zone!!!
4138	*/
4139	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4140	{
4141	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4142	return VINF_SUCCESS;
4143
4144	#ifdef VBOX_STRICT
4145	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4146	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4147	{
4148	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4149	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4150	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4151	}
4152	#endif
4153
4154	int rc;
4155	PVM pVM = pVCpu->CTX_SUFF(pVM);
4156	bool fSteppingDB = false;
4157	bool fInterceptMovDRx = false;
4158	if (pVCpu->hm.s.fSingleInstruction)
4159	{
4160	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4161	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4162	{
4163	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4164	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4165	AssertRCReturn(rc, rc);
4166	Assert(fSteppingDB == false);
4167	}
4168	else
4169	{
4170	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4171	pVCpu->hm.s.fClearTrapFlag = true;
4172	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4173	fSteppingDB = true;
4174	}
4175	}
4176
4177	if ( fSteppingDB
4178	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4179	{
4180	/*
4181	* Use the combined guest and host DRx values found in the hypervisor
4182	* register set because the debugger has breakpoints active or someone
4183	* is single stepping on the host side without a monitor trap flag.
4184	*
4185	* Note! DBGF expects a clean DR6 state before executing guest code.
4186	*/
4187	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4188	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4189	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4190	{
4191	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4192	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4193	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4194	}
4195	else
4196	#endif
4197	if (!CPUMIsHyperDebugStateActive(pVCpu))
4198	{
4199	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4200	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4201	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4202	}
4203
4204	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4205	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4206	AssertRCReturn(rc, rc);
4207
4208	pVCpu->hm.s.fUsingHyperDR7 = true;
4209	fInterceptMovDRx = true;
4210	}
4211	else
4212	{
4213	/*
4214	* If the guest has enabled debug registers, we need to load them prior to
4215	* executing guest code so they'll trigger at the right time.
4216	*/
4217	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4218	{
4219	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4220	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4221	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4222	{
4223	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4224	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4225	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4226	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4227	}
4228	else
4229	#endif
4230	if (!CPUMIsGuestDebugStateActive(pVCpu))
4231	{
4232	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4233	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4234	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4235	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4236	}
4237	Assert(!fInterceptMovDRx);
4238	}
4239	/*
4240	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4241	* must intercept #DB in order to maintain a correct DR6 guest value, and
4242	* because we need to intercept it to prevent nested #DBs from hanging the
4243	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4244	*/
4245	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4246	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4247	&& !CPUMIsGuestDebugStateActive(pVCpu))
4248	#else
4249	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4250	#endif
4251	{
4252	fInterceptMovDRx = true;
4253	}
4254
4255	/* Update guest DR7. */
4256	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4257	AssertRCReturn(rc, rc);
4258
4259	pVCpu->hm.s.fUsingHyperDR7 = false;
4260	}
4261
4262	/*
4263	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4264	*/
4265	if (fInterceptMovDRx)
4266	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4267	else
4268	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4269	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4270	AssertRCReturn(rc, rc);
4271
4272	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4273	return VINF_SUCCESS;
4274	}
4275
4276
4277	#ifdef VBOX_STRICT
4278	/**
4279	* Strict function to validate segment registers.
4280	*
4281	* @remarks ASSUMES CR0 is up to date.
4282	*/
4283	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4284	{
4285	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4286	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4287	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4288	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4289	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4290	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4291	{
4292	/* Protected mode checks */
4293	/* CS */
4294	Assert(pCtx->cs.Attr.n.u1Present);
4295	Assert(!(pCtx->cs.Attr.u & 0xf00));
4296	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4297	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4298	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4299	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4300	\|\| (pCtx->cs.Attr.n.u1Granularity));
4301	/* CS cannot be loaded with NULL in protected mode. */
4302	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4303	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4304	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4305	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4306	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4307	else
4308	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4309	/* SS */
4310	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4311	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4312	if ( !(pCtx->cr0 & X86_CR0_PE)
4313	\|\| pCtx->cs.Attr.n.u4Type == 3)
4314	{
4315	Assert(!pCtx->ss.Attr.n.u2Dpl);
4316	}
4317	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4318	{
4319	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4320	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4321	Assert(pCtx->ss.Attr.n.u1Present);
4322	Assert(!(pCtx->ss.Attr.u & 0xf00));
4323	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4324	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4325	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4326	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4327	\|\| (pCtx->ss.Attr.n.u1Granularity));
4328	}
4329	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4330	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4331	{
4332	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4333	Assert(pCtx->ds.Attr.n.u1Present);
4334	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4335	Assert(!(pCtx->ds.Attr.u & 0xf00));
4336	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4337	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4338	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4339	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4340	\|\| (pCtx->ds.Attr.n.u1Granularity));
4341	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4342	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4343	}
4344	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4345	{
4346	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4347	Assert(pCtx->es.Attr.n.u1Present);
4348	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4349	Assert(!(pCtx->es.Attr.u & 0xf00));
4350	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4351	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4352	\|\| !(pCtx->es.Attr.n.u1Granularity));
4353	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4354	\|\| (pCtx->es.Attr.n.u1Granularity));
4355	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4356	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4357	}
4358	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4359	{
4360	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4361	Assert(pCtx->fs.Attr.n.u1Present);
4362	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4363	Assert(!(pCtx->fs.Attr.u & 0xf00));
4364	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4365	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4366	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4367	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4368	\|\| (pCtx->fs.Attr.n.u1Granularity));
4369	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4370	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4371	}
4372	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4373	{
4374	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4375	Assert(pCtx->gs.Attr.n.u1Present);
4376	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4377	Assert(!(pCtx->gs.Attr.u & 0xf00));
4378	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4379	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4380	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4381	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4382	\|\| (pCtx->gs.Attr.n.u1Granularity));
4383	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4384	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4385	}
4386	/* 64-bit capable CPUs. */
4387	# if HC_ARCH_BITS == 64
4388	Assert(!(pCtx->cs.u64Base >> 32));
4389	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4390	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4391	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4392	# endif
4393	}
4394	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4395	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4396	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4397	{
4398	/* Real and v86 mode checks. */
4399	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4400	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4401	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4402	{
4403	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4404	}
4405	else
4406	{
4407	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4408	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4409	}
4410
4411	/* CS */
4412	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4413	Assert(pCtx->cs.u32Limit == 0xffff);
4414	Assert(u32CSAttr == 0xf3);
4415	/* SS */
4416	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4417	Assert(pCtx->ss.u32Limit == 0xffff);
4418	Assert(u32SSAttr == 0xf3);
4419	/* DS */
4420	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4421	Assert(pCtx->ds.u32Limit == 0xffff);
4422	Assert(u32DSAttr == 0xf3);
4423	/* ES */
4424	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4425	Assert(pCtx->es.u32Limit == 0xffff);
4426	Assert(u32ESAttr == 0xf3);
4427	/* FS */
4428	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4429	Assert(pCtx->fs.u32Limit == 0xffff);
4430	Assert(u32FSAttr == 0xf3);
4431	/* GS */
4432	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4433	Assert(pCtx->gs.u32Limit == 0xffff);
4434	Assert(u32GSAttr == 0xf3);
4435	/* 64-bit capable CPUs. */
4436	# if HC_ARCH_BITS == 64
4437	Assert(!(pCtx->cs.u64Base >> 32));
4438	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4439	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4440	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4441	# endif
4442	}
4443	}
4444	#endif /* VBOX_STRICT */
4445
4446
4447	/**
4448	* Writes a guest segment register into the guest-state area in the VMCS.
4449	*
4450	* @returns VBox status code.
4451	* @param pVCpu The cross context virtual CPU structure.
4452	* @param idxSel Index of the selector in the VMCS.
4453	* @param idxLimit Index of the segment limit in the VMCS.
4454	* @param idxBase Index of the segment base in the VMCS.
4455	* @param idxAccess Index of the access rights of the segment in the VMCS.
4456	* @param pSelReg Pointer to the segment selector.
4457	*
4458	* @remarks No-long-jump zone!!!
4459	*/
4460	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4461	uint32_t idxAccess, PCPUMSELREG pSelReg)
4462	{
4463	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4464	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4465	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4466	AssertRCReturn(rc, rc);
4467
4468	uint32_t u32Access = pSelReg->Attr.u;
4469	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4470	{
4471	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4472	u32Access = 0xf3;
4473	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4474	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4475	}
4476	else
4477	{
4478	/*
4479	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4480	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4481	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4482	* loaded in protected-mode have their attribute as 0.
4483	*/
4484	if (!u32Access)
4485	u32Access = X86DESCATTR_UNUSABLE;
4486	}
4487
4488	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4489	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4490	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4491
4492	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4493	AssertRCReturn(rc, rc);
4494	return rc;
4495	}
4496
4497
4498	/**
4499	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4500	* into the guest-state area in the VMCS.
4501	*
4502	* @returns VBox status code.
4503	* @param pVCpu The cross context virtual CPU structure.
4504	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4505	* out-of-sync. Make sure to update the required fields
4506	* before using them.
4507	*
4508	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4509	* @remarks No-long-jump zone!!!
4510	*/
4511	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4512	{
4513	int rc = VERR_INTERNAL_ERROR_5;
4514	PVM pVM = pVCpu->CTX_SUFF(pVM);
4515
4516	/*
4517	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4518	*/
4519	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4520	{
4521	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4522	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4523	{
4524	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4525	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4526	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4527	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4528	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4529	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4530	}
4531
4532	#ifdef VBOX_WITH_REM
4533	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4534	{
4535	Assert(pVM->hm.s.vmx.pRealModeTSS);
4536	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4537	if ( pVCpu->hm.s.vmx.fWasInRealMode
4538	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4539	{
4540	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4541	in real-mode (e.g. OpenBSD 4.0) */
4542	REMFlushTBs(pVM);
4543	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4544	pVCpu->hm.s.vmx.fWasInRealMode = false;
4545	}
4546	}
4547	#endif
4548	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4549	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4550	AssertRCReturn(rc, rc);
4551	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4552	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4553	AssertRCReturn(rc, rc);
4554	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4555	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4556	AssertRCReturn(rc, rc);
4557	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4558	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4559	AssertRCReturn(rc, rc);
4560	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4561	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4562	AssertRCReturn(rc, rc);
4563	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4564	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4565	AssertRCReturn(rc, rc);
4566
4567	#ifdef VBOX_STRICT
4568	/* Validate. */
4569	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4570	#endif
4571
4572	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4573	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4574	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4575	}
4576
4577	/*
4578	* Guest TR.
4579	*/
4580	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4581	{
4582	/*
4583	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4584	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4585	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4586	*/
4587	uint16_t u16Sel = 0;
4588	uint32_t u32Limit = 0;
4589	uint64_t u64Base = 0;
4590	uint32_t u32AccessRights = 0;
4591
4592	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4593	{
4594	u16Sel = pMixedCtx->tr.Sel;
4595	u32Limit = pMixedCtx->tr.u32Limit;
4596	u64Base = pMixedCtx->tr.u64Base;
4597	u32AccessRights = pMixedCtx->tr.Attr.u;
4598	}
4599	else
4600	{
4601	Assert(pVM->hm.s.vmx.pRealModeTSS);
4602	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4603
4604	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4605	RTGCPHYS GCPhys;
4606	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4607	AssertRCReturn(rc, rc);
4608
4609	X86DESCATTR DescAttr;
4610	DescAttr.u = 0;
4611	DescAttr.n.u1Present = 1;
4612	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4613
4614	u16Sel = 0;
4615	u32Limit = HM_VTX_TSS_SIZE;
4616	u64Base = GCPhys; /* in real-mode phys = virt. */
4617	u32AccessRights = DescAttr.u;
4618	}
4619
4620	/* Validate. */
4621	Assert(!(u16Sel & RT_BIT(2)));
4622	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4623	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4624	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4625	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4626	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4627	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4628	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4629	Assert( (u32Limit & 0xfff) == 0xfff
4630	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4631	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4632	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4633
4634	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4635	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4636	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4637	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4638	AssertRCReturn(rc, rc);
4639
4640	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4641	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4642	}
4643
4644	/*
4645	* Guest GDTR.
4646	*/
4647	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4648	{
4649	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4650	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4651	AssertRCReturn(rc, rc);
4652
4653	/* Validate. */
4654	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4655
4656	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4657	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4658	}
4659
4660	/*
4661	* Guest LDTR.
4662	*/
4663	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4664	{
4665	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4666	uint32_t u32Access = 0;
4667	if (!pMixedCtx->ldtr.Attr.u)
4668	u32Access = X86DESCATTR_UNUSABLE;
4669	else
4670	u32Access = pMixedCtx->ldtr.Attr.u;
4671
4672	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4673	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4674	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4675	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4676	AssertRCReturn(rc, rc);
4677
4678	/* Validate. */
4679	if (!(u32Access & X86DESCATTR_UNUSABLE))
4680	{
4681	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4682	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4683	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4684	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4685	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4686	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4687	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4688	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4689	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4690	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4691	}
4692
4693	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4694	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4695	}
4696
4697	/*
4698	* Guest IDTR.
4699	*/
4700	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4701	{
4702	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4703	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4704	AssertRCReturn(rc, rc);
4705
4706	/* Validate. */
4707	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4708
4709	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4710	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4711	}
4712
4713	return VINF_SUCCESS;
4714	}
4715
4716
4717	/**
4718	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4719	* areas.
4720	*
4721	* These MSRs will automatically be loaded to the host CPU on every successful
4722	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4723	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4724	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4725	*
4726	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4727	*
4728	* @returns VBox status code.
4729	* @param pVCpu The cross context virtual CPU structure.
4730	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4731	* out-of-sync. Make sure to update the required fields
4732	* before using them.
4733	*
4734	* @remarks No-long-jump zone!!!
4735	*/
4736	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4737	{
4738	AssertPtr(pVCpu);
4739	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4740
4741	/*
4742	* MSRs that we use the auto-load/store MSR area in the VMCS.
4743	*/
4744	PVM pVM = pVCpu->CTX_SUFF(pVM);
4745	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4746	{
4747	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4748	#if HC_ARCH_BITS == 32
4749	if (pVM->hm.s.fAllow64BitGuests)
4750	{
4751	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4752	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4753	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4754	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4755	AssertRCReturn(rc, rc);
4756	# ifdef LOG_ENABLED
4757	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4758	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4759	{
4760	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4761	pMsr->u64Value));
4762	}
4763	# endif
4764	}
4765	#endif
4766	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4767	}
4768
4769	/*
4770	* Guest Sysenter MSRs.
4771	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4772	* VM-exits on WRMSRs for these MSRs.
4773	*/
4774	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4775	{
4776	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4777	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4778	}
4779
4780	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4781	{
4782	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4783	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4784	}
4785
4786	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4787	{
4788	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4789	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4790	}
4791
4792	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4793	{
4794	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4795	{
4796	/*
4797	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4798	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4799	*/
4800	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4801	{
4802	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4803	AssertRCReturn(rc,rc);
4804	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4805	}
4806	else
4807	{
4808	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4809	NULL /* pfAddedAndUpdated */);
4810	AssertRCReturn(rc, rc);
4811
4812	/* We need to intercept reads too, see @bugref{7386#c16}. */
4813	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4814	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4815	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4816	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4817	}
4818	}
4819	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4820	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4821	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4822	}
4823
4824	return VINF_SUCCESS;
4825	}
4826
4827
4828	/**
4829	* Loads the guest activity state into the guest-state area in the VMCS.
4830	*
4831	* @returns VBox status code.
4832	* @param pVCpu The cross context virtual CPU structure.
4833	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4834	* out-of-sync. Make sure to update the required fields
4835	* before using them.
4836	*
4837	* @remarks No-long-jump zone!!!
4838	*/
4839	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4840	{
4841	NOREF(pMixedCtx);
4842	/** @todo See if we can make use of other states, e.g.
4843	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4844	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4845	{
4846	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4847	AssertRCReturn(rc, rc);
4848
4849	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4850	}
4851	return VINF_SUCCESS;
4852	}
4853
4854
4855	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4856	/**
4857	* Check if guest state allows safe use of 32-bit switcher again.
4858	*
4859	* Segment bases and protected mode structures must be 32-bit addressable
4860	* because the 32-bit switcher will ignore high dword when writing these VMCS
4861	* fields. See @bugref{8432} for details.
4862	*
4863	* @returns true if safe, false if must continue to use the 64-bit switcher.
4864	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4865	* out-of-sync. Make sure to update the required fields
4866	* before using them.
4867	*
4868	* @remarks No-long-jump zone!!!
4869	*/
4870	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4871	{
4872	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4873	return false;
4874	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4875	return false;
4876	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4877	return false;
4878	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4879	return false;
4880	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4881	return false;
4882	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4883	return false;
4884	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4885	return false;
4886	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4887	return false;
4888	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4889	return false;
4890	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4891	return false;
4892	/* All good, bases are 32-bit. */
4893	return true;
4894	}
4895	#endif
4896
4897
4898	/**
4899	* Sets up the appropriate function to run guest code.
4900	*
4901	* @returns VBox status code.
4902	* @param pVCpu The cross context virtual CPU structure.
4903	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4904	* out-of-sync. Make sure to update the required fields
4905	* before using them.
4906	*
4907	* @remarks No-long-jump zone!!!
4908	*/
4909	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4910	{
4911	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4912	{
4913	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4914	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4915	#endif
4916	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4917	#if HC_ARCH_BITS == 32
4918	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4919	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4920	{
4921	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4922	{
4923	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4924	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4925	\| HM_CHANGED_VMX_ENTRY_CTLS
4926	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4927	}
4928	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4929
4930	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4931	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4932	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4933	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4934	}
4935	#else
4936	/* 64-bit host. */
4937	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4938	#endif
4939	}
4940	else
4941	{
4942	/* Guest is not in long mode, use the 32-bit handler. */
4943	#if HC_ARCH_BITS == 32
4944	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4945	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4946	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4947	{
4948	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4949	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4950	\| HM_CHANGED_VMX_ENTRY_CTLS
4951	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4952	}
4953	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4954	/*
4955	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4956	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4957	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4958	* the much faster 32-bit switcher again.
4959	*/
4960	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4961	{
4962	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4963	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4964	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4965	}
4966	else
4967	{
4968	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4969	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4970	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4971	{
4972	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4973	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4974	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4975	\| HM_CHANGED_VMX_ENTRY_CTLS
4976	\| HM_CHANGED_VMX_EXIT_CTLS
4977	\| HM_CHANGED_HOST_CONTEXT);
4978	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4979	}
4980	}
4981	# else
4982	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4983	# endif
4984	#else
4985	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4986	#endif
4987	}
4988	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4989	return VINF_SUCCESS;
4990	}
4991
4992
4993	/**
4994	* Wrapper for running the guest code in VT-x.
4995	*
4996	* @returns VBox status code, no informational status codes.
4997	* @param pVM The cross context VM structure.
4998	* @param pVCpu The cross context virtual CPU structure.
4999	* @param pCtx Pointer to the guest-CPU context.
5000	*
5001	* @remarks No-long-jump zone!!!
5002	*/
5003	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5004	{
5005	/*
5006	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5007	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5008	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5009	*/
5010	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5011	/** @todo Add stats for resume vs launch. */
5012	#ifdef VBOX_WITH_KERNEL_USING_XMM
5013	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5014	#else
5015	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5016	#endif
5017	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5018	return rc;
5019	}
5020
5021
5022	/**
5023	* Reports world-switch error and dumps some useful debug info.
5024	*
5025	* @param pVM The cross context VM structure.
5026	* @param pVCpu The cross context virtual CPU structure.
5027	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5028	* @param pCtx Pointer to the guest-CPU context.
5029	* @param pVmxTransient Pointer to the VMX transient structure (only
5030	* exitReason updated).
5031	*/
5032	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5033	{
5034	Assert(pVM);
5035	Assert(pVCpu);
5036	Assert(pCtx);
5037	Assert(pVmxTransient);
5038	HMVMX_ASSERT_PREEMPT_SAFE();
5039
5040	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5041	switch (rcVMRun)
5042	{
5043	case VERR_VMX_INVALID_VMXON_PTR:
5044	AssertFailed();
5045	break;
5046	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5047	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5048	{
5049	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5050	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5051	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5052	AssertRC(rc);
5053
5054	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5055	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5056	Cannot do it here as we may have been long preempted. */
5057
5058	#ifdef VBOX_STRICT
5059	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5060	pVmxTransient->uExitReason));
5061	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5062	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5063	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5064	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5065	else
5066	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5067	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5068	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5069
5070	/* VMX control bits. */
5071	uint32_t u32Val;
5072	uint64_t u64Val;
5073	RTHCUINTREG uHCReg;
5074	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5075	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5076	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5077	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5078	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5079	{
5080	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5081	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5082	}
5083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5084	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5085	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5086	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5088	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5090	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5091	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5092	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5094	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5096	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5097	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5098	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5100	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5102	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5104	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5106	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5108	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5109	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5110	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5111	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5112	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5113	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5114	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5115	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5116	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5117	if (pVM->hm.s.fNestedPaging)
5118	{
5119	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5120	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5121	}
5122
5123	/* Guest bits. */
5124	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5125	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5126	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5127	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5128	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5129	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5130	if (pVM->hm.s.vmx.fVpid)
5131	{
5132	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5133	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5134	}
5135
5136	/* Host bits. */
5137	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5138	Log4(("Host CR0 %#RHr\n", uHCReg));
5139	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5140	Log4(("Host CR3 %#RHr\n", uHCReg));
5141	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5142	Log4(("Host CR4 %#RHr\n", uHCReg));
5143
5144	RTGDTR HostGdtr;
5145	PCX86DESCHC pDesc;
5146	ASMGetGDTR(&HostGdtr);
5147	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5148	Log4(("Host CS %#08x\n", u32Val));
5149	if (u32Val < HostGdtr.cbGdt)
5150	{
5151	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5152	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5153	}
5154
5155	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5156	Log4(("Host DS %#08x\n", u32Val));
5157	if (u32Val < HostGdtr.cbGdt)
5158	{
5159	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5160	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5161	}
5162
5163	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5164	Log4(("Host ES %#08x\n", u32Val));
5165	if (u32Val < HostGdtr.cbGdt)
5166	{
5167	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5168	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5169	}
5170
5171	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5172	Log4(("Host FS %#08x\n", u32Val));
5173	if (u32Val < HostGdtr.cbGdt)
5174	{
5175	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5176	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5177	}
5178
5179	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5180	Log4(("Host GS %#08x\n", u32Val));
5181	if (u32Val < HostGdtr.cbGdt)
5182	{
5183	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5184	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5185	}
5186
5187	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5188	Log4(("Host SS %#08x\n", u32Val));
5189	if (u32Val < HostGdtr.cbGdt)
5190	{
5191	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5192	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5193	}
5194
5195	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5196	Log4(("Host TR %#08x\n", u32Val));
5197	if (u32Val < HostGdtr.cbGdt)
5198	{
5199	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5200	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5201	}
5202
5203	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5204	Log4(("Host TR Base %#RHv\n", uHCReg));
5205	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5206	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5207	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5208	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5209	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5210	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5211	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5212	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5213	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5214	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5215	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5216	Log4(("Host RSP %#RHv\n", uHCReg));
5217	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5218	Log4(("Host RIP %#RHv\n", uHCReg));
5219	# if HC_ARCH_BITS == 64
5220	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5221	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5222	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5223	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5224	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5225	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5226	# endif
5227	#endif /* VBOX_STRICT */
5228	break;
5229	}
5230
5231	default:
5232	/* Impossible */
5233	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5234	break;
5235	}
5236	NOREF(pVM); NOREF(pCtx);
5237	}
5238
5239
5240	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5241	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5242	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5243	#endif
5244	#ifdef VBOX_STRICT
5245	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5246	{
5247	switch (idxField)
5248	{
5249	case VMX_VMCS_GUEST_RIP:
5250	case VMX_VMCS_GUEST_RSP:
5251	case VMX_VMCS_GUEST_SYSENTER_EIP:
5252	case VMX_VMCS_GUEST_SYSENTER_ESP:
5253	case VMX_VMCS_GUEST_GDTR_BASE:
5254	case VMX_VMCS_GUEST_IDTR_BASE:
5255	case VMX_VMCS_GUEST_CS_BASE:
5256	case VMX_VMCS_GUEST_DS_BASE:
5257	case VMX_VMCS_GUEST_ES_BASE:
5258	case VMX_VMCS_GUEST_FS_BASE:
5259	case VMX_VMCS_GUEST_GS_BASE:
5260	case VMX_VMCS_GUEST_SS_BASE:
5261	case VMX_VMCS_GUEST_LDTR_BASE:
5262	case VMX_VMCS_GUEST_TR_BASE:
5263	case VMX_VMCS_GUEST_CR3:
5264	return true;
5265	}
5266	return false;
5267	}
5268
5269	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5270	{
5271	switch (idxField)
5272	{
5273	/* Read-only fields. */
5274	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5275	return true;
5276	}
5277	/* Remaining readable fields should also be writable. */
5278	return hmR0VmxIsValidWriteField(idxField);
5279	}
5280	#endif /* VBOX_STRICT */
5281
5282
5283	/**
5284	* Executes the specified handler in 64-bit mode.
5285	*
5286	* @returns VBox status code (no informational status codes).
5287	* @param pVM The cross context VM structure.
5288	* @param pVCpu The cross context virtual CPU structure.
5289	* @param pCtx Pointer to the guest CPU context.
5290	* @param enmOp The operation to perform.
5291	* @param cParams Number of parameters.
5292	* @param paParam Array of 32-bit parameters.
5293	*/
5294	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5295	uint32_t cParams, uint32_t *paParam)
5296	{
5297	NOREF(pCtx);
5298
5299	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5300	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5301	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5302	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5303
5304	#ifdef VBOX_STRICT
5305	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5306	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5307
5308	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5309	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5310	#endif
5311
5312	/* Disable interrupts. */
5313	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5314
5315	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5316	RTCPUID idHostCpu = RTMpCpuId();
5317	CPUMR0SetLApic(pVCpu, idHostCpu);
5318	#endif
5319
5320	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5321	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5322
5323	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5324	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5325	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5326
5327	/* Leave VMX Root Mode. */
5328	VMXDisable();
5329
5330	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5331
5332	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5333	CPUMSetHyperEIP(pVCpu, enmOp);
5334	for (int i = (int)cParams - 1; i >= 0; i--)
5335	CPUMPushHyper(pVCpu, paParam[i]);
5336
5337	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5338
5339	/* Call the switcher. */
5340	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5341	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5342
5343	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5344	/* Make sure the VMX instructions don't cause #UD faults. */
5345	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5346
5347	/* Re-enter VMX Root Mode */
5348	int rc2 = VMXEnable(HCPhysCpuPage);
5349	if (RT_FAILURE(rc2))
5350	{
5351	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5352	ASMSetFlags(fOldEFlags);
5353	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5354	return rc2;
5355	}
5356
5357	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5358	AssertRC(rc2);
5359	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5360	Assert(!(ASMGetFlags() & X86_EFL_IF));
5361	ASMSetFlags(fOldEFlags);
5362	return rc;
5363	}
5364
5365
5366	/**
5367	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5368	* supporting 64-bit guests.
5369	*
5370	* @returns VBox status code.
5371	* @param fResume Whether to VMLAUNCH or VMRESUME.
5372	* @param pCtx Pointer to the guest-CPU context.
5373	* @param pCache Pointer to the VMCS cache.
5374	* @param pVM The cross context VM structure.
5375	* @param pVCpu The cross context virtual CPU structure.
5376	*/
5377	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5378	{
5379	NOREF(fResume);
5380
5381	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5382	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5383
5384	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5385	pCache->uPos = 1;
5386	pCache->interPD = PGMGetInterPaeCR3(pVM);
5387	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5388	#endif
5389
5390	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5391	pCache->TestIn.HCPhysCpuPage = 0;
5392	pCache->TestIn.HCPhysVmcs = 0;
5393	pCache->TestIn.pCache = 0;
5394	pCache->TestOut.HCPhysVmcs = 0;
5395	pCache->TestOut.pCache = 0;
5396	pCache->TestOut.pCtx = 0;
5397	pCache->TestOut.eflags = 0;
5398	#else
5399	NOREF(pCache);
5400	#endif
5401
5402	uint32_t aParam[10];
5403	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5404	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5405	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5406	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5407	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5408	aParam[5] = 0;
5409	aParam[6] = VM_RC_ADDR(pVM, pVM);
5410	aParam[7] = 0;
5411	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5412	aParam[9] = 0;
5413
5414	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5415	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5416	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5417	#endif
5418	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5419
5420	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5421	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5422	Assert(pCtx->dr[4] == 10);
5423	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5424	#endif
5425
5426	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5427	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5428	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5429	pVCpu->hm.s.vmx.HCPhysVmcs));
5430	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5431	pCache->TestOut.HCPhysVmcs));
5432	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5433	pCache->TestOut.pCache));
5434	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5435	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5436	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5437	pCache->TestOut.pCtx));
5438	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5439	#endif
5440	return rc;
5441	}
5442
5443
5444	/**
5445	* Initialize the VMCS-Read cache.
5446	*
5447	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5448	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5449	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5450	* (those that have a 32-bit FULL & HIGH part).
5451	*
5452	* @returns VBox status code.
5453	* @param pVM The cross context VM structure.
5454	* @param pVCpu The cross context virtual CPU structure.
5455	*/
5456	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5457	{
5458	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5459	{ \
5460	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5461	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5462	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5463	++cReadFields; \
5464	}
5465
5466	AssertPtr(pVM);
5467	AssertPtr(pVCpu);
5468	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5469	uint32_t cReadFields = 0;
5470
5471	/*
5472	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5473	* and serve to indicate exceptions to the rules.
5474	*/
5475
5476	/* Guest-natural selector base fields. */
5477	#if 0
5478	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5479	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5480	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5481	#endif
5482	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5483	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5484	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5485	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5486	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5487	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5488	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5489	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5490	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5491	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5492	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5493	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5494	#if 0
5495	/* Unused natural width guest-state fields. */
5496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5498	#endif
5499	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5500	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5501
5502	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5503	#if 0
5504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5511	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5512	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5513	#endif
5514
5515	/* Natural width guest-state fields. */
5516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5517	#if 0
5518	/* Currently unused field. */
5519	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5520	#endif
5521
5522	if (pVM->hm.s.fNestedPaging)
5523	{
5524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5525	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5526	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5527	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5528	}
5529	else
5530	{
5531	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5532	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5533	}
5534
5535	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5536	return VINF_SUCCESS;
5537	}
5538
5539
5540	/**
5541	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5542	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5543	* darwin, running 64-bit guests).
5544	*
5545	* @returns VBox status code.
5546	* @param pVCpu The cross context virtual CPU structure.
5547	* @param idxField The VMCS field encoding.
5548	* @param u64Val 16, 32 or 64-bit value.
5549	*/
5550	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5551	{
5552	int rc;
5553	switch (idxField)
5554	{
5555	/*
5556	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5557	*/
5558	/* 64-bit Control fields. */
5559	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5560	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5561	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5562	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5563	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5564	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5565	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5566	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5567	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5568	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5569	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5570	case VMX_VMCS64_CTRL_EPTP_FULL:
5571	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5572	/* 64-bit Guest-state fields. */
5573	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5574	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5575	case VMX_VMCS64_GUEST_PAT_FULL:
5576	case VMX_VMCS64_GUEST_EFER_FULL:
5577	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5578	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5579	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5580	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5581	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5582	/* 64-bit Host-state fields. */
5583	case VMX_VMCS64_HOST_PAT_FULL:
5584	case VMX_VMCS64_HOST_EFER_FULL:
5585	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5586	{
5587	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5588	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5589	break;
5590	}
5591
5592	/*
5593	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5594	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5595	*/
5596	/* Natural-width Guest-state fields. */
5597	case VMX_VMCS_GUEST_CR3:
5598	case VMX_VMCS_GUEST_ES_BASE:
5599	case VMX_VMCS_GUEST_CS_BASE:
5600	case VMX_VMCS_GUEST_SS_BASE:
5601	case VMX_VMCS_GUEST_DS_BASE:
5602	case VMX_VMCS_GUEST_FS_BASE:
5603	case VMX_VMCS_GUEST_GS_BASE:
5604	case VMX_VMCS_GUEST_LDTR_BASE:
5605	case VMX_VMCS_GUEST_TR_BASE:
5606	case VMX_VMCS_GUEST_GDTR_BASE:
5607	case VMX_VMCS_GUEST_IDTR_BASE:
5608	case VMX_VMCS_GUEST_RSP:
5609	case VMX_VMCS_GUEST_RIP:
5610	case VMX_VMCS_GUEST_SYSENTER_ESP:
5611	case VMX_VMCS_GUEST_SYSENTER_EIP:
5612	{
5613	if (!(RT_HI_U32(u64Val)))
5614	{
5615	/* If this field is 64-bit, VT-x will zero out the top bits. */
5616	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5617	}
5618	else
5619	{
5620	/* Assert that only the 32->64 switcher case should ever come here. */
5621	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5622	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5623	}
5624	break;
5625	}
5626
5627	default:
5628	{
5629	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5630	rc = VERR_INVALID_PARAMETER;
5631	break;
5632	}
5633	}
5634	AssertRCReturn(rc, rc);
5635	return rc;
5636	}
5637
5638
5639	/**
5640	* Queue up a VMWRITE by using the VMCS write cache.
5641	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5642	*
5643	* @param pVCpu The cross context virtual CPU structure.
5644	* @param idxField The VMCS field encoding.
5645	* @param u64Val 16, 32 or 64-bit value.
5646	*/
5647	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5648	{
5649	AssertPtr(pVCpu);
5650	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5651
5652	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5653	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5654
5655	/* Make sure there are no duplicates. */
5656	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5657	{
5658	if (pCache->Write.aField[i] == idxField)
5659	{
5660	pCache->Write.aFieldVal[i] = u64Val;
5661	return VINF_SUCCESS;
5662	}
5663	}
5664
5665	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5666	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5667	pCache->Write.cValidEntries++;
5668	return VINF_SUCCESS;
5669	}
5670	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5671
5672
5673	/**
5674	* Sets up the usage of TSC-offsetting and updates the VMCS.
5675	*
5676	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5677	* VMX preemption timer.
5678	*
5679	* @returns VBox status code.
5680	* @param pVM The cross context VM structure.
5681	* @param pVCpu The cross context virtual CPU structure.
5682	*
5683	* @remarks No-long-jump zone!!!
5684	*/
5685	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5686	{
5687	int rc;
5688	bool fOffsettedTsc;
5689	bool fParavirtTsc;
5690	if (pVM->hm.s.vmx.fUsePreemptTimer)
5691	{
5692	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5693	&fOffsettedTsc, &fParavirtTsc);
5694
5695	/* Make sure the returned values have sane upper and lower boundaries. */
5696	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5697	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5698	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5699	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5700
5701	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5702	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5703	}
5704	else
5705	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5706
5707	/** @todo later optimize this to be done elsewhere and not before every
5708	* VM-entry. */
5709	if (fParavirtTsc)
5710	{
5711	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5712	information before every VM-entry, hence disable it for performance sake. */
5713	#if 0
5714	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5715	AssertRC(rc);
5716	#endif
5717	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5718	}
5719
5720	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5721	{
5722	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5723	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5724
5725	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5726	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5727	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5728	}
5729	else
5730	{
5731	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5732	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5733	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5734	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5735	}
5736	}
5737
5738
5739	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5740	/**
5741	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5742	* VM-exit interruption info type.
5743	*
5744	* @returns The IEM exception flags.
5745	* @param uVector The event vector.
5746	* @param uVmxVectorType The VMX event type.
5747	*
5748	* @remarks This function currently only constructs flags required for
5749	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5750	* and CR2 aspects of an exception are not included).
5751	*/
5752	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5753	{
5754	uint32_t fIemXcptFlags;
5755	switch (uVmxVectorType)
5756	{
5757	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5758	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5759	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5760	break;
5761
5762	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5763	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5764	break;
5765
5766	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5767	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5768	break;
5769
5770	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5771	{
5772	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5773	if (uVector == X86_XCPT_BP)
5774	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5775	else if (uVector == X86_XCPT_OF)
5776	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5777	else
5778	{
5779	fIemXcptFlags = 0;
5780	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5781	}
5782	break;
5783	}
5784
5785	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5786	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5787	break;
5788
5789	default:
5790	fIemXcptFlags = 0;
5791	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5792	break;
5793	}
5794	return fIemXcptFlags;
5795	}
5796
5797	#else
5798	/**
5799	* Determines if an exception is a contributory exception.
5800	*
5801	* Contributory exceptions are ones which can cause double-faults unless the
5802	* original exception was a benign exception. Page-fault is intentionally not
5803	* included here as it's a conditional contributory exception.
5804	*
5805	* @returns true if the exception is contributory, false otherwise.
5806	* @param uVector The exception vector.
5807	*/
5808	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5809	{
5810	switch (uVector)
5811	{
5812	case X86_XCPT_GP:
5813	case X86_XCPT_SS:
5814	case X86_XCPT_NP:
5815	case X86_XCPT_TS:
5816	case X86_XCPT_DE:
5817	return true;
5818	default:
5819	break;
5820	}
5821	return false;
5822	}
5823	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5824
5825
5826	/**
5827	* Sets an event as a pending event to be injected into the guest.
5828	*
5829	* @param pVCpu The cross context virtual CPU structure.
5830	* @param u32IntInfo The VM-entry interruption-information field.
5831	* @param cbInstr The VM-entry instruction length in bytes (for software
5832	* interrupts, exceptions and privileged software
5833	* exceptions).
5834	* @param u32ErrCode The VM-entry exception error code.
5835	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5836	* page-fault.
5837	*
5838	* @remarks Statistics counter assumes this is a guest event being injected or
5839	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5840	* always incremented.
5841	*/
5842	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5843	RTGCUINTPTR GCPtrFaultAddress)
5844	{
5845	Assert(!pVCpu->hm.s.Event.fPending);
5846	pVCpu->hm.s.Event.fPending = true;
5847	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5848	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5849	pVCpu->hm.s.Event.cbInstr = cbInstr;
5850	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5851	}
5852
5853
5854	/**
5855	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5856	*
5857	* @param pVCpu The cross context virtual CPU structure.
5858	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5859	* out-of-sync. Make sure to update the required fields
5860	* before using them.
5861	*/
5862	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5863	{
5864	NOREF(pMixedCtx);
5865	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5866	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5867	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5868	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5869	}
5870
5871
5872	/**
5873	* Handle a condition that occurred while delivering an event through the guest
5874	* IDT.
5875	*
5876	* @returns Strict VBox status code (i.e. informational status codes too).
5877	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5878	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5879	* to continue execution of the guest which will delivery the \#DF.
5880	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5881	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5882	*
5883	* @param pVCpu The cross context virtual CPU structure.
5884	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5885	* out-of-sync. Make sure to update the required fields
5886	* before using them.
5887	* @param pVmxTransient Pointer to the VMX transient structure.
5888	*
5889	* @remarks No-long-jump zone!!!
5890	*/
5891	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5892	{
5893	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5894
5895	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5896	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5897
5898	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5899	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5900	{
5901	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5902	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5903	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5904	/*
5905	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5906	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5907	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5908	* exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5909	* currently has no way of storing that these exceptions were caused by these instructions
5910	* (ICEBP's #DB poses the problem).
5911	*/
5912	IEMXCPTRAISE enmRaise;
5913	IEMXCPTRAISEINFO fRaiseInfo;
5914	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5915	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5916	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5917	{
5918	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5919	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5920	}
5921	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5922	{
5923	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5924	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5925	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5926	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5927	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5928	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5929	uExitVectorType), VERR_VMX_IPE_5);
5930	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5931
5932	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5933	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5934	{
5935	pVmxTransient->fVectoringPF = true;
5936	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5937	}
5938	}
5939	else
5940	{
5941	/*
5942	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5943	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5944	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5945	*/
5946	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5947	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5948	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5949	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5950	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5951	}
5952
5953	/*
5954	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5955	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5956	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5957	* subsequent VM-entry would fail.
5958	*
5959	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5960	*/
5961	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5962	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5963	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5964	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5965	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5966	{
5967	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5968	}
5969
5970	switch (enmRaise)
5971	{
5972	case IEMXCPTRAISE_CURRENT_XCPT:
5973	{
5974	Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
5975	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5976	Assert(rcStrict == VINF_SUCCESS);
5977	break;
5978	}
5979
5980	case IEMXCPTRAISE_PREV_EVENT:
5981	{
5982	uint32_t u32ErrCode;
5983	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5984	{
5985	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5986	AssertRCReturn(rc2, rc2);
5987	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5988	}
5989	else
5990	u32ErrCode = 0;
5991
5992	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5993	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5994	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5995	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5996
5997	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
5998	pVCpu->hm.s.Event.u32ErrCode));
5999	Assert(rcStrict == VINF_SUCCESS);
6000	break;
6001	}
6002
6003	case IEMXCPTRAISE_REEXEC_INSTR:
6004	Assert(rcStrict == VINF_SUCCESS);
6005	break;
6006
6007	case IEMXCPTRAISE_DOUBLE_FAULT:
6008	{
6009	/*
6010	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6011	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6012	*/
6013	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6014	{
6015	pVmxTransient->fVectoringDoublePF = true;
6016	Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6017	pMixedCtx->cr2));
6018	rcStrict = VINF_SUCCESS;
6019	}
6020	else
6021	{
6022	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6023	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6024	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6025	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6026	rcStrict = VINF_HM_DOUBLE_FAULT;
6027	}
6028	break;
6029	}
6030
6031	case IEMXCPTRAISE_TRIPLE_FAULT:
6032	{
6033	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6034	uExitVector));
6035	rcStrict = VINF_EM_RESET;
6036	break;
6037	}
6038
6039	case IEMXCPTRAISE_CPU_HANG:
6040	{
6041	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6042	rcStrict = VERR_EM_GUEST_CPU_HANG;
6043	break;
6044	}
6045
6046	default:
6047	{
6048	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6049	rcStrict = VERR_VMX_IPE_2;
6050	break;
6051	}
6052	}
6053	#else
6054	typedef enum
6055	{
6056	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6057	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6058	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6059	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6060	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6061	} VMXREFLECTXCPT;
6062
6063	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6064	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6065	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6066	{
6067	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6068	{
6069	enmReflect = VMXREFLECTXCPT_XCPT;
6070	#ifdef VBOX_STRICT
6071	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6072	&& uExitVector == X86_XCPT_PF)
6073	{
6074	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6075	}
6076	#endif
6077	if ( uExitVector == X86_XCPT_PF
6078	&& uIdtVector == X86_XCPT_PF)
6079	{
6080	pVmxTransient->fVectoringDoublePF = true;
6081	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6082	}
6083	else if ( uExitVector == X86_XCPT_AC
6084	&& uIdtVector == X86_XCPT_AC)
6085	{
6086	enmReflect = VMXREFLECTXCPT_HANG;
6087	Log4(("IDT: Nested #AC - Bad guest\n"));
6088	}
6089	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6090	&& hmR0VmxIsContributoryXcpt(uExitVector)
6091	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6092	\|\| uIdtVector == X86_XCPT_PF))
6093	{
6094	enmReflect = VMXREFLECTXCPT_DF;
6095	}
6096	else if (uIdtVector == X86_XCPT_DF)
6097	enmReflect = VMXREFLECTXCPT_TF;
6098	}
6099	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6100	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6101	{
6102	/*
6103	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6104	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6105	*/
6106	enmReflect = VMXREFLECTXCPT_XCPT;
6107
6108	if (uExitVector == X86_XCPT_PF)
6109	{
6110	pVmxTransient->fVectoringPF = true;
6111	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6112	}
6113	}
6114	}
6115	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6116	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6117	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6118	{
6119	/*
6120	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6121	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6122	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6123	*/
6124	enmReflect = VMXREFLECTXCPT_XCPT;
6125	}
6126
6127	/*
6128	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6129	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6130	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6131	*
6132	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6133	*/
6134	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6135	&& enmReflect == VMXREFLECTXCPT_XCPT
6136	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6137	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6138	{
6139	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6140	}
6141
6142	switch (enmReflect)
6143	{
6144	case VMXREFLECTXCPT_XCPT:
6145	{
6146	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6147	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6148	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6149
6150	uint32_t u32ErrCode = 0;
6151	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6152	{
6153	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6154	AssertRCReturn(rc2, rc2);
6155	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6156	}
6157
6158	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6159	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6160	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6161	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6162	rcStrict = VINF_SUCCESS;
6163	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6164	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6165
6166	break;
6167	}
6168
6169	case VMXREFLECTXCPT_DF:
6170	{
6171	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6172	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6173	rcStrict = VINF_HM_DOUBLE_FAULT;
6174	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6175	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6176
6177	break;
6178	}
6179
6180	case VMXREFLECTXCPT_TF:
6181	{
6182	rcStrict = VINF_EM_RESET;
6183	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6184	uExitVector));
6185	break;
6186	}
6187
6188	case VMXREFLECTXCPT_HANG:
6189	{
6190	rcStrict = VERR_EM_GUEST_CPU_HANG;
6191	break;
6192	}
6193
6194	default:
6195	Assert(rcStrict == VINF_SUCCESS);
6196	break;
6197	}
6198	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6199	}
6200	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6201	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6202	&& uExitVector != X86_XCPT_DF
6203	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6204	{
6205	/*
6206	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6207	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6208	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6209	*/
6210	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6211	{
6212	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6213	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6214	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6215	}
6216	}
6217
6218	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6219	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6220	return rcStrict;
6221	}
6222
6223
6224	/**
6225	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6226	*
6227	* @returns VBox status code.
6228	* @param pVCpu The cross context virtual CPU structure.
6229	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6230	* out-of-sync. Make sure to update the required fields
6231	* before using them.
6232	*
6233	* @remarks No-long-jump zone!!!
6234	*/
6235	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6236	{
6237	NOREF(pMixedCtx);
6238
6239	/*
6240	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6241	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6242	*/
6243	VMMRZCallRing3Disable(pVCpu);
6244	HM_DISABLE_PREEMPT();
6245
6246	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6247	{
6248	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6249	* and 'dbgc-init' containing:
6250	* sxe "xcpt_de"
6251	* sxe "xcpt_bp"
6252	* sxi "xcpt_gp"
6253	* sxi "xcpt_ss"
6254	* sxi "xcpt_np"
6255	*/
6256	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6257	#endif
6258	uint32_t uVal = 0;
6259	uint32_t uShadow = 0;
6260	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6261	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6262	AssertRCReturn(rc, rc);
6263
6264	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6265	CPUMSetGuestCR0(pVCpu, uVal);
6266	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6267	}
6268
6269	HM_RESTORE_PREEMPT();
6270	VMMRZCallRing3Enable(pVCpu);
6271	return VINF_SUCCESS;
6272	}
6273
6274
6275	/**
6276	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6277	*
6278	* @returns VBox status code.
6279	* @param pVCpu The cross context virtual CPU structure.
6280	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6281	* out-of-sync. Make sure to update the required fields
6282	* before using them.
6283	*
6284	* @remarks No-long-jump zone!!!
6285	*/
6286	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6287	{
6288	NOREF(pMixedCtx);
6289
6290	int rc = VINF_SUCCESS;
6291	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6292	{
6293	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6294	uint32_t uVal = 0;
6295	uint32_t uShadow = 0;
6296	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6297	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6298	AssertRCReturn(rc, rc);
6299
6300	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6301	CPUMSetGuestCR4(pVCpu, uVal);
6302	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6303	}
6304	return rc;
6305	}
6306
6307
6308	/**
6309	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6310	*
6311	* @returns VBox status code.
6312	* @param pVCpu The cross context virtual CPU structure.
6313	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6314	* out-of-sync. Make sure to update the required fields
6315	* before using them.
6316	*
6317	* @remarks No-long-jump zone!!!
6318	*/
6319	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6320	{
6321	int rc = VINF_SUCCESS;
6322	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6323	{
6324	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6325	uint64_t u64Val = 0;
6326	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6327	AssertRCReturn(rc, rc);
6328
6329	pMixedCtx->rip = u64Val;
6330	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6331	}
6332	return rc;
6333	}
6334
6335
6336	/**
6337	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6338	*
6339	* @returns VBox status code.
6340	* @param pVCpu The cross context virtual CPU structure.
6341	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6342	* out-of-sync. Make sure to update the required fields
6343	* before using them.
6344	*
6345	* @remarks No-long-jump zone!!!
6346	*/
6347	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6348	{
6349	int rc = VINF_SUCCESS;
6350	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6351	{
6352	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6353	uint64_t u64Val = 0;
6354	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6355	AssertRCReturn(rc, rc);
6356
6357	pMixedCtx->rsp = u64Val;
6358	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6359	}
6360	return rc;
6361	}
6362
6363
6364	/**
6365	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6366	*
6367	* @returns VBox status code.
6368	* @param pVCpu The cross context virtual CPU structure.
6369	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6370	* out-of-sync. Make sure to update the required fields
6371	* before using them.
6372	*
6373	* @remarks No-long-jump zone!!!
6374	*/
6375	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6376	{
6377	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6378	{
6379	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6380	uint32_t uVal = 0;
6381	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6382	AssertRCReturn(rc, rc);
6383
6384	pMixedCtx->eflags.u32 = uVal;
6385	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6386	{
6387	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6388	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6389
6390	pMixedCtx->eflags.Bits.u1VM = 0;
6391	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6392	}
6393
6394	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6395	}
6396	return VINF_SUCCESS;
6397	}
6398
6399
6400	/**
6401	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6402	* guest-CPU context.
6403	*/
6404	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6405	{
6406	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6407	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6408	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6409	return rc;
6410	}
6411
6412
6413	/**
6414	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6415	* from the guest-state area in the VMCS.
6416	*
6417	* @param pVCpu The cross context virtual CPU structure.
6418	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6419	* out-of-sync. Make sure to update the required fields
6420	* before using them.
6421	*
6422	* @remarks No-long-jump zone!!!
6423	*/
6424	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6425	{
6426	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6427	{
6428	uint32_t uIntrState = 0;
6429	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6430	AssertRC(rc);
6431
6432	if (!uIntrState)
6433	{
6434	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6435	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6436
6437	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6438	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6439	}
6440	else
6441	{
6442	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6443	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6444	{
6445	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6446	AssertRC(rc);
6447	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6448	AssertRC(rc);
6449
6450	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6451	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6452	}
6453	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6454	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6455
6456	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6457	{
6458	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6459	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6460	}
6461	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6462	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6463	}
6464
6465	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6466	}
6467	}
6468
6469
6470	/**
6471	* Saves the guest's activity state.
6472	*
6473	* @returns VBox status code.
6474	* @param pVCpu The cross context virtual CPU structure.
6475	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6476	* out-of-sync. Make sure to update the required fields
6477	* before using them.
6478	*
6479	* @remarks No-long-jump zone!!!
6480	*/
6481	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6482	{
6483	NOREF(pMixedCtx);
6484	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6485	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6486	return VINF_SUCCESS;
6487	}
6488
6489
6490	/**
6491	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6492	* the current VMCS into the guest-CPU context.
6493	*
6494	* @returns VBox status code.
6495	* @param pVCpu The cross context virtual CPU structure.
6496	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6497	* out-of-sync. Make sure to update the required fields
6498	* before using them.
6499	*
6500	* @remarks No-long-jump zone!!!
6501	*/
6502	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6503	{
6504	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6505	{
6506	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6507	uint32_t u32Val = 0;
6508	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6509	pMixedCtx->SysEnter.cs = u32Val;
6510	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6511	}
6512
6513	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6514	{
6515	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6516	uint64_t u64Val = 0;
6517	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6518	pMixedCtx->SysEnter.eip = u64Val;
6519	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6520	}
6521	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6522	{
6523	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6524	uint64_t u64Val = 0;
6525	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6526	pMixedCtx->SysEnter.esp = u64Val;
6527	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6528	}
6529	return VINF_SUCCESS;
6530	}
6531
6532
6533	/**
6534	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6535	* the CPU back into the guest-CPU context.
6536	*
6537	* @returns VBox status code.
6538	* @param pVCpu The cross context virtual CPU structure.
6539	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6540	* out-of-sync. Make sure to update the required fields
6541	* before using them.
6542	*
6543	* @remarks No-long-jump zone!!!
6544	*/
6545	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6546	{
6547	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6548	VMMRZCallRing3Disable(pVCpu);
6549	HM_DISABLE_PREEMPT();
6550
6551	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6552	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6553	{
6554	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6555	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6556	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6557	}
6558
6559	HM_RESTORE_PREEMPT();
6560	VMMRZCallRing3Enable(pVCpu);
6561
6562	return VINF_SUCCESS;
6563	}
6564
6565
6566	/**
6567	* Saves the auto load/store'd guest MSRs from the current VMCS into
6568	* the guest-CPU context.
6569	*
6570	* @returns VBox status code.
6571	* @param pVCpu The cross context virtual CPU structure.
6572	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6573	* out-of-sync. Make sure to update the required fields
6574	* before using them.
6575	*
6576	* @remarks No-long-jump zone!!!
6577	*/
6578	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6579	{
6580	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6581	return VINF_SUCCESS;
6582
6583	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6584	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6585	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6586	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6587	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6588	{
6589	switch (pMsr->u32Msr)
6590	{
6591	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6592	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6593	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6594	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6595	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6596	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6597	break;
6598
6599	default:
6600	{
6601	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6602	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6603	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6604	}
6605	}
6606	}
6607
6608	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6609	return VINF_SUCCESS;
6610	}
6611
6612
6613	/**
6614	* Saves the guest control registers from the current VMCS into the guest-CPU
6615	* context.
6616	*
6617	* @returns VBox status code.
6618	* @param pVCpu The cross context virtual CPU structure.
6619	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6620	* out-of-sync. Make sure to update the required fields
6621	* before using them.
6622	*
6623	* @remarks No-long-jump zone!!!
6624	*/
6625	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6626	{
6627	/* Guest CR0. Guest FPU. */
6628	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6629	AssertRCReturn(rc, rc);
6630
6631	/* Guest CR4. */
6632	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6633	AssertRCReturn(rc, rc);
6634
6635	/* Guest CR2 - updated always during the world-switch or in #PF. */
6636	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6637	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6638	{
6639	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6640	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6641	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6642
6643	PVM pVM = pVCpu->CTX_SUFF(pVM);
6644	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6645	\|\| ( pVM->hm.s.fNestedPaging
6646	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6647	{
6648	uint64_t u64Val = 0;
6649	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6650	if (pMixedCtx->cr3 != u64Val)
6651	{
6652	CPUMSetGuestCR3(pVCpu, u64Val);
6653	if (VMMRZCallRing3IsEnabled(pVCpu))
6654	{
6655	PGMUpdateCR3(pVCpu, u64Val);
6656	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6657	}
6658	else
6659	{
6660	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6661	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6662	}
6663	}
6664
6665	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6666	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6667	{
6668	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6669	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6670	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6671	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6672	AssertRCReturn(rc, rc);
6673
6674	if (VMMRZCallRing3IsEnabled(pVCpu))
6675	{
6676	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6677	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6678	}
6679	else
6680	{
6681	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6682	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6683	}
6684	}
6685	}
6686
6687	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6688	}
6689
6690	/*
6691	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6692	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6693	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6694	*
6695	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6696	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6697	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6698	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6699	*
6700	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6701	*/
6702	if (VMMRZCallRing3IsEnabled(pVCpu))
6703	{
6704	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6705	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6706
6707	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6708	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6709
6710	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6711	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6712	}
6713
6714	return rc;
6715	}
6716
6717
6718	/**
6719	* Reads a guest segment register from the current VMCS into the guest-CPU
6720	* context.
6721	*
6722	* @returns VBox status code.
6723	* @param pVCpu The cross context virtual CPU structure.
6724	* @param idxSel Index of the selector in the VMCS.
6725	* @param idxLimit Index of the segment limit in the VMCS.
6726	* @param idxBase Index of the segment base in the VMCS.
6727	* @param idxAccess Index of the access rights of the segment in the VMCS.
6728	* @param pSelReg Pointer to the segment selector.
6729	*
6730	* @remarks No-long-jump zone!!!
6731	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6732	* macro as that takes care of whether to read from the VMCS cache or
6733	* not.
6734	*/
6735	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6736	PCPUMSELREG pSelReg)
6737	{
6738	NOREF(pVCpu);
6739
6740	uint32_t u32Val = 0;
6741	int rc = VMXReadVmcs32(idxSel, &u32Val);
6742	AssertRCReturn(rc, rc);
6743	pSelReg->Sel = (uint16_t)u32Val;
6744	pSelReg->ValidSel = (uint16_t)u32Val;
6745	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6746
6747	rc = VMXReadVmcs32(idxLimit, &u32Val);
6748	AssertRCReturn(rc, rc);
6749	pSelReg->u32Limit = u32Val;
6750
6751	uint64_t u64Val = 0;
6752	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6753	AssertRCReturn(rc, rc);
6754	pSelReg->u64Base = u64Val;
6755
6756	rc = VMXReadVmcs32(idxAccess, &u32Val);
6757	AssertRCReturn(rc, rc);
6758	pSelReg->Attr.u = u32Val;
6759
6760	/*
6761	* If VT-x marks the segment as unusable, most other bits remain undefined:
6762	* - For CS the L, D and G bits have meaning.
6763	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6764	* - For the remaining data segments no bits are defined.
6765	*
6766	* The present bit and the unusable bit has been observed to be set at the
6767	* same time (the selector was supposed to be invalid as we started executing
6768	* a V8086 interrupt in ring-0).
6769	*
6770	* What should be important for the rest of the VBox code, is that the P bit is
6771	* cleared. Some of the other VBox code recognizes the unusable bit, but
6772	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6773	* safe side here, we'll strip off P and other bits we don't care about. If
6774	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6775	*
6776	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6777	*/
6778	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6779	{
6780	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6781
6782	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6783	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6784	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6785
6786	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6787	#ifdef DEBUG_bird
6788	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6789	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6790	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6791	#endif
6792	}
6793	return VINF_SUCCESS;
6794	}
6795
6796
6797	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6798	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6799	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6800	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6801	#else
6802	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6803	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6804	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6805	#endif
6806
6807
6808	/**
6809	* Saves the guest segment registers from the current VMCS into the guest-CPU
6810	* context.
6811	*
6812	* @returns VBox status code.
6813	* @param pVCpu The cross context virtual CPU structure.
6814	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6815	* out-of-sync. Make sure to update the required fields
6816	* before using them.
6817	*
6818	* @remarks No-long-jump zone!!!
6819	*/
6820	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6821	{
6822	/* Guest segment registers. */
6823	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6824	{
6825	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6826	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6827	AssertRCReturn(rc, rc);
6828
6829	rc = VMXLOCAL_READ_SEG(CS, cs);
6830	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6831	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6832	rc \|= VMXLOCAL_READ_SEG(ES, es);
6833	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6834	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6835	AssertRCReturn(rc, rc);
6836
6837	/* Restore segment attributes for real-on-v86 mode hack. */
6838	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6839	{
6840	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6841	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6842	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6843	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6844	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6845	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6846	}
6847	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6848	}
6849
6850	return VINF_SUCCESS;
6851	}
6852
6853
6854	/**
6855	* Saves the guest descriptor table registers and task register from the current
6856	* VMCS into the guest-CPU context.
6857	*
6858	* @returns VBox status code.
6859	* @param pVCpu The cross context virtual CPU structure.
6860	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6861	* out-of-sync. Make sure to update the required fields
6862	* before using them.
6863	*
6864	* @remarks No-long-jump zone!!!
6865	*/
6866	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6867	{
6868	int rc = VINF_SUCCESS;
6869
6870	/* Guest LDTR. */
6871	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6872	{
6873	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6874	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6875	AssertRCReturn(rc, rc);
6876	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6877	}
6878
6879	/* Guest GDTR. */
6880	uint64_t u64Val = 0;
6881	uint32_t u32Val = 0;
6882	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6883	{
6884	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6885	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6886	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6887	pMixedCtx->gdtr.pGdt = u64Val;
6888	pMixedCtx->gdtr.cbGdt = u32Val;
6889	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6890	}
6891
6892	/* Guest IDTR. */
6893	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6894	{
6895	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6896	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6897	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6898	pMixedCtx->idtr.pIdt = u64Val;
6899	pMixedCtx->idtr.cbIdt = u32Val;
6900	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6901	}
6902
6903	/* Guest TR. */
6904	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6905	{
6906	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6907	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6908	AssertRCReturn(rc, rc);
6909
6910	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6911	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6912	{
6913	rc = VMXLOCAL_READ_SEG(TR, tr);
6914	AssertRCReturn(rc, rc);
6915	}
6916	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6917	}
6918	return rc;
6919	}
6920
6921	#undef VMXLOCAL_READ_SEG
6922
6923
6924	/**
6925	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6926	* context.
6927	*
6928	* @returns VBox status code.
6929	* @param pVCpu The cross context virtual CPU structure.
6930	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6931	* out-of-sync. Make sure to update the required fields
6932	* before using them.
6933	*
6934	* @remarks No-long-jump zone!!!
6935	*/
6936	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6937	{
6938	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6939	{
6940	if (!pVCpu->hm.s.fUsingHyperDR7)
6941	{
6942	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6943	uint32_t u32Val;
6944	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6945	pMixedCtx->dr[7] = u32Val;
6946	}
6947
6948	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6949	}
6950	return VINF_SUCCESS;
6951	}
6952
6953
6954	/**
6955	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6956	*
6957	* @returns VBox status code.
6958	* @param pVCpu The cross context virtual CPU structure.
6959	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6960	* out-of-sync. Make sure to update the required fields
6961	* before using them.
6962	*
6963	* @remarks No-long-jump zone!!!
6964	*/
6965	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6966	{
6967	NOREF(pMixedCtx);
6968
6969	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6970	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6971	return VINF_SUCCESS;
6972	}
6973
6974
6975	/**
6976	* Saves the entire guest state from the currently active VMCS into the
6977	* guest-CPU context.
6978	*
6979	* This essentially VMREADs all guest-data.
6980	*
6981	* @returns VBox status code.
6982	* @param pVCpu The cross context virtual CPU structure.
6983	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6984	* out-of-sync. Make sure to update the required fields
6985	* before using them.
6986	*/
6987	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6988	{
6989	Assert(pVCpu);
6990	Assert(pMixedCtx);
6991
6992	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6993	return VINF_SUCCESS;
6994
6995	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6996	again on the ring-3 callback path, there is no real need to. */
6997	if (VMMRZCallRing3IsEnabled(pVCpu))
6998	VMMR0LogFlushDisable(pVCpu);
6999	else
7000	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7001	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7002
7003	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7004	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7005
7006	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7007	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7008
7009	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7010	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7011
7012	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7013	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7014
7015	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7016	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7017
7018	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7019	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7020
7021	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7022	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7023
7024	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7025	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7026
7027	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7028	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7029
7030	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7031	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7032
7033	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7034	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7035	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7036
7037	if (VMMRZCallRing3IsEnabled(pVCpu))
7038	VMMR0LogFlushEnable(pVCpu);
7039
7040	return VINF_SUCCESS;
7041	}
7042
7043
7044	/**
7045	* Saves basic guest registers needed for IEM instruction execution.
7046	*
7047	* @returns VBox status code (OR-able).
7048	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7049	* @param pMixedCtx Pointer to the CPU context of the guest.
7050	* @param fMemory Whether the instruction being executed operates on
7051	* memory or not. Only CR0 is synced up if clear.
7052	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7053	*/
7054	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7055	{
7056	/*
7057	* We assume all general purpose registers other than RSP are available.
7058	*
7059	* - RIP is a must, as it will be incremented or otherwise changed.
7060	* - RFLAGS are always required to figure the CPL.
7061	* - RSP isn't always required, however it's a GPR, so frequently required.
7062	* - SS and CS are the only segment register needed if IEM doesn't do memory
7063	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7064	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7065	* be required for memory accesses.
7066	*
7067	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7068	*/
7069	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7070	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7071	if (fNeedRsp)
7072	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7073	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7074	if (!fMemory)
7075	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7076	else
7077	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7078	AssertRCReturn(rc, rc);
7079	return rc;
7080	}
7081
7082
7083	/**
7084	* Ensures that we've got a complete basic guest-context.
7085	*
7086	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7087	* is for the interpreter.
7088	*
7089	* @returns VBox status code.
7090	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7091	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7092	* needing to be synced in.
7093	* @thread EMT(pVCpu)
7094	*/
7095	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7096	{
7097	/* Note! Since this is only applicable to VT-x, the implementation is placed
7098	in the VT-x part of the sources instead of the generic stuff. */
7099	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7100	{
7101	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7102	/*
7103	* For now, imply that the caller might change everything too. Do this after
7104	* saving the guest state so as to not trigger assertions.
7105	*/
7106	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7107	return rc;
7108	}
7109	return VINF_SUCCESS;
7110	}
7111
7112
7113	/**
7114	* Check per-VM and per-VCPU force flag actions that require us to go back to
7115	* ring-3 for one reason or another.
7116	*
7117	* @returns Strict VBox status code (i.e. informational status codes too)
7118	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7119	* ring-3.
7120	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7121	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7122	* interrupts)
7123	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7124	* all EMTs to be in ring-3.
7125	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7126	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7127	* to the EM loop.
7128	*
7129	* @param pVM The cross context VM structure.
7130	* @param pVCpu The cross context virtual CPU structure.
7131	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7132	* out-of-sync. Make sure to update the required fields
7133	* before using them.
7134	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7135	*/
7136	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7137	{
7138	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7139
7140	/*
7141	* Anything pending? Should be more likely than not if we're doing a good job.
7142	*/
7143	if ( !fStepping
7144	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7145	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7146	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7147	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7148	return VINF_SUCCESS;
7149
7150	/* We need the control registers now, make sure the guest-CPU context is updated. */
7151	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7152	AssertRCReturn(rc3, rc3);
7153
7154	/* Pending HM CR3 sync. */
7155	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7156	{
7157	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7158	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7159	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7160	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7161	}
7162
7163	/* Pending HM PAE PDPEs. */
7164	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7165	{
7166	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7167	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7168	}
7169
7170	/* Pending PGM C3 sync. */
7171	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7172	{
7173	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7174	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7175	if (rcStrict2 != VINF_SUCCESS)
7176	{
7177	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7178	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7179	return rcStrict2;
7180	}
7181	}
7182
7183	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7184	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7185	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7186	{
7187	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7188	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7189	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7190	return rc2;
7191	}
7192
7193	/* Pending VM request packets, such as hardware interrupts. */
7194	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7195	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7196	{
7197	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7198	return VINF_EM_PENDING_REQUEST;
7199	}
7200
7201	/* Pending PGM pool flushes. */
7202	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7203	{
7204	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7205	return VINF_PGM_POOL_FLUSH_PENDING;
7206	}
7207
7208	/* Pending DMA requests. */
7209	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7210	{
7211	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7212	return VINF_EM_RAW_TO_R3;
7213	}
7214
7215	return VINF_SUCCESS;
7216	}
7217
7218
7219	/**
7220	* Converts any TRPM trap into a pending HM event. This is typically used when
7221	* entering from ring-3 (not longjmp returns).
7222	*
7223	* @param pVCpu The cross context virtual CPU structure.
7224	*/
7225	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7226	{
7227	Assert(TRPMHasTrap(pVCpu));
7228	Assert(!pVCpu->hm.s.Event.fPending);
7229
7230	uint8_t uVector;
7231	TRPMEVENT enmTrpmEvent;
7232	RTGCUINT uErrCode;
7233	RTGCUINTPTR GCPtrFaultAddress;
7234	uint8_t cbInstr;
7235
7236	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7237	AssertRC(rc);
7238
7239	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7240	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7241	if (enmTrpmEvent == TRPM_TRAP)
7242	{
7243	switch (uVector)
7244	{
7245	case X86_XCPT_NMI:
7246	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7247	break;
7248
7249	case X86_XCPT_BP:
7250	case X86_XCPT_OF:
7251	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7252	break;
7253
7254	case X86_XCPT_PF:
7255	case X86_XCPT_DF:
7256	case X86_XCPT_TS:
7257	case X86_XCPT_NP:
7258	case X86_XCPT_SS:
7259	case X86_XCPT_GP:
7260	case X86_XCPT_AC:
7261	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7262	/* fall thru */
7263	default:
7264	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7265	break;
7266	}
7267	}
7268	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7269	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7270	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7271	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7272	else
7273	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7274
7275	rc = TRPMResetTrap(pVCpu);
7276	AssertRC(rc);
7277	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7278	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7279
7280	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7281	}
7282
7283
7284	/**
7285	* Converts the pending HM event into a TRPM trap.
7286	*
7287	* @param pVCpu The cross context virtual CPU structure.
7288	*/
7289	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7290	{
7291	Assert(pVCpu->hm.s.Event.fPending);
7292
7293	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7294	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7295	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7296	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7297
7298	/* If a trap was already pending, we did something wrong! */
7299	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7300
7301	TRPMEVENT enmTrapType;
7302	switch (uVectorType)
7303	{
7304	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7305	enmTrapType = TRPM_HARDWARE_INT;
7306	break;
7307
7308	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7309	enmTrapType = TRPM_SOFTWARE_INT;
7310	break;
7311
7312	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7313	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7314	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7315	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7316	enmTrapType = TRPM_TRAP;
7317	break;
7318
7319	default:
7320	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7321	enmTrapType = TRPM_32BIT_HACK;
7322	break;
7323	}
7324
7325	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7326
7327	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7328	AssertRC(rc);
7329
7330	if (fErrorCodeValid)
7331	TRPMSetErrorCode(pVCpu, uErrorCode);
7332
7333	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7334	&& uVector == X86_XCPT_PF)
7335	{
7336	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7337	}
7338	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7339	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7340	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7341	{
7342	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7343	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7344	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7345	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7346	}
7347
7348	/* Clear any pending events from the VMCS. */
7349	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7350	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7351
7352	/* We're now done converting the pending event. */
7353	pVCpu->hm.s.Event.fPending = false;
7354	}
7355
7356
7357	/**
7358	* Does the necessary state syncing before returning to ring-3 for any reason
7359	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7360	*
7361	* @returns VBox status code.
7362	* @param pVCpu The cross context virtual CPU structure.
7363	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7364	* be out-of-sync. Make sure to update the required
7365	* fields before using them.
7366	* @param fSaveGuestState Whether to save the guest state or not.
7367	*
7368	* @remarks No-long-jmp zone!!!
7369	*/
7370	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7371	{
7372	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7373	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7374
7375	RTCPUID idCpu = RTMpCpuId();
7376	Log4Func(("HostCpuId=%u\n", idCpu));
7377
7378	/*
7379	* !!! IMPORTANT !!!
7380	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7381	*/
7382
7383	/* Save the guest state if necessary. */
7384	if ( fSaveGuestState
7385	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7386	{
7387	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7388	AssertRCReturn(rc, rc);
7389	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7390	}
7391
7392	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7393	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7394	{
7395	/* We shouldn't reload CR0 without saving it first. */
7396	if (!fSaveGuestState)
7397	{
7398	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7399	AssertRCReturn(rc, rc);
7400	}
7401	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7402	}
7403
7404	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7405	#ifdef VBOX_STRICT
7406	if (CPUMIsHyperDebugStateActive(pVCpu))
7407	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7408	#endif
7409	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7410	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7411	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7412	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7413
7414	#if HC_ARCH_BITS == 64
7415	/* Restore host-state bits that VT-x only restores partially. */
7416	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7417	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7418	{
7419	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7420	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7421	}
7422	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7423	#endif
7424
7425	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7426	if (pVCpu->hm.s.vmx.fLazyMsrs)
7427	{
7428	/* We shouldn't reload the guest MSRs without saving it first. */
7429	if (!fSaveGuestState)
7430	{
7431	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7432	AssertRCReturn(rc, rc);
7433	}
7434	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7435	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7436	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7437	}
7438
7439	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7440	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7441
7442	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7443	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7444	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7445	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7446	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7447	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7448	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7449	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7450
7451	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7452
7453	/** @todo This partially defeats the purpose of having preemption hooks.
7454	* The problem is, deregistering the hooks should be moved to a place that
7455	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7456	* context.
7457	*/
7458	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7459	{
7460	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7461	AssertRCReturn(rc, rc);
7462
7463	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7464	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7465	}
7466	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7467	NOREF(idCpu);
7468
7469	return VINF_SUCCESS;
7470	}
7471
7472
7473	/**
7474	* Leaves the VT-x session.
7475	*
7476	* @returns VBox status code.
7477	* @param pVCpu The cross context virtual CPU structure.
7478	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7479	* out-of-sync. Make sure to update the required fields
7480	* before using them.
7481	*
7482	* @remarks No-long-jmp zone!!!
7483	*/
7484	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7485	{
7486	HM_DISABLE_PREEMPT();
7487	HMVMX_ASSERT_CPU_SAFE();
7488	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7489	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7490
7491	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7492	and done this from the VMXR0ThreadCtxCallback(). */
7493	if (!pVCpu->hm.s.fLeaveDone)
7494	{
7495	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7496	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7497	pVCpu->hm.s.fLeaveDone = true;
7498	}
7499	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7500
7501	/*
7502	* !!! IMPORTANT !!!
7503	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7504	*/
7505
7506	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7507	/** @todo Deregistering here means we need to VMCLEAR always
7508	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7509	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7510	VMMR0ThreadCtxHookDisable(pVCpu);
7511
7512	/* Leave HM context. This takes care of local init (term). */
7513	int rc = HMR0LeaveCpu(pVCpu);
7514
7515	HM_RESTORE_PREEMPT();
7516	return rc;
7517	}
7518
7519
7520	/**
7521	* Does the necessary state syncing before doing a longjmp to ring-3.
7522	*
7523	* @returns VBox status code.
7524	* @param pVCpu The cross context virtual CPU structure.
7525	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7526	* out-of-sync. Make sure to update the required fields
7527	* before using them.
7528	*
7529	* @remarks No-long-jmp zone!!!
7530	*/
7531	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7532	{
7533	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7534	}
7535
7536
7537	/**
7538	* Take necessary actions before going back to ring-3.
7539	*
7540	* An action requires us to go back to ring-3. This function does the necessary
7541	* steps before we can safely return to ring-3. This is not the same as longjmps
7542	* to ring-3, this is voluntary and prepares the guest so it may continue
7543	* executing outside HM (recompiler/IEM).
7544	*
7545	* @returns VBox status code.
7546	* @param pVM The cross context VM structure.
7547	* @param pVCpu The cross context virtual CPU structure.
7548	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7549	* out-of-sync. Make sure to update the required fields
7550	* before using them.
7551	* @param rcExit The reason for exiting to ring-3. Can be
7552	* VINF_VMM_UNKNOWN_RING3_CALL.
7553	*/
7554	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7555	{
7556	Assert(pVM);
7557	Assert(pVCpu);
7558	Assert(pMixedCtx);
7559	HMVMX_ASSERT_PREEMPT_SAFE();
7560
7561	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7562	{
7563	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7564	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7565	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7566	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7567	}
7568
7569	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7570	VMMRZCallRing3Disable(pVCpu);
7571	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7572
7573	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7574	if (pVCpu->hm.s.Event.fPending)
7575	{
7576	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7577	Assert(!pVCpu->hm.s.Event.fPending);
7578	}
7579
7580	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7581	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7582
7583	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7584	and if we're injecting an event we should have a TRPM trap pending. */
7585	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7586	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7587	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7588	#endif
7589
7590	/* Save guest state and restore host state bits. */
7591	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7592	AssertRCReturn(rc, rc);
7593	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7594	/* Thread-context hooks are unregistered at this point!!! */
7595
7596	/* Sync recompiler state. */
7597	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7598	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7599	\| CPUM_CHANGED_LDTR
7600	\| CPUM_CHANGED_GDTR
7601	\| CPUM_CHANGED_IDTR
7602	\| CPUM_CHANGED_TR
7603	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7604	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7605	if ( pVM->hm.s.fNestedPaging
7606	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7607	{
7608	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7609	}
7610
7611	Assert(!pVCpu->hm.s.fClearTrapFlag);
7612
7613	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7614	if (rcExit != VINF_EM_RAW_INTERRUPT)
7615	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7616
7617	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7618
7619	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7620	VMMRZCallRing3RemoveNotification(pVCpu);
7621	VMMRZCallRing3Enable(pVCpu);
7622
7623	return rc;
7624	}
7625
7626
7627	/**
7628	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7629	* longjump to ring-3 and possibly get preempted.
7630	*
7631	* @returns VBox status code.
7632	* @param pVCpu The cross context virtual CPU structure.
7633	* @param enmOperation The operation causing the ring-3 longjump.
7634	* @param pvUser Opaque pointer to the guest-CPU context. The data
7635	* may be out-of-sync. Make sure to update the required
7636	* fields before using them.
7637	*/
7638	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7639	{
7640	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7641	{
7642	/*
7643	* !!! IMPORTANT !!!
7644	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7645	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7646	*/
7647	VMMRZCallRing3RemoveNotification(pVCpu);
7648	VMMRZCallRing3Disable(pVCpu);
7649	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7650	RTThreadPreemptDisable(&PreemptState);
7651
7652	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7653	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7654
7655	#if HC_ARCH_BITS == 64
7656	/* Restore host-state bits that VT-x only restores partially. */
7657	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7658	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7659	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7660	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7661	#endif
7662	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7663	if (pVCpu->hm.s.vmx.fLazyMsrs)
7664	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7665
7666	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7667	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7668	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7669	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7670	{
7671	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7672	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7673	}
7674
7675	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7676	VMMR0ThreadCtxHookDisable(pVCpu);
7677	HMR0LeaveCpu(pVCpu);
7678	RTThreadPreemptRestore(&PreemptState);
7679	return VINF_SUCCESS;
7680	}
7681
7682	Assert(pVCpu);
7683	Assert(pvUser);
7684	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7685	HMVMX_ASSERT_PREEMPT_SAFE();
7686
7687	VMMRZCallRing3Disable(pVCpu);
7688	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7689
7690	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7691	enmOperation));
7692
7693	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7694	AssertRCReturn(rc, rc);
7695
7696	VMMRZCallRing3Enable(pVCpu);
7697	return VINF_SUCCESS;
7698	}
7699
7700
7701	/**
7702	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7703	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7704	*
7705	* @param pVCpu The cross context virtual CPU structure.
7706	*/
7707	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7708	{
7709	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7710	{
7711	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7712	{
7713	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7714	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7715	AssertRC(rc);
7716	Log4(("Setup interrupt-window exiting\n"));
7717	}
7718	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7719	}
7720
7721
7722	/**
7723	* Clears the interrupt-window exiting control in the VMCS.
7724	*
7725	* @param pVCpu The cross context virtual CPU structure.
7726	*/
7727	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7728	{
7729	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7730	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7731	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7732	AssertRC(rc);
7733	Log4(("Cleared interrupt-window exiting\n"));
7734	}
7735
7736
7737	/**
7738	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7739	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7740	*
7741	* @param pVCpu The cross context virtual CPU structure.
7742	*/
7743	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7744	{
7745	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7746	{
7747	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7748	{
7749	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7750	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7751	AssertRC(rc);
7752	Log4(("Setup NMI-window exiting\n"));
7753	}
7754	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7755	}
7756
7757
7758	/**
7759	* Clears the NMI-window exiting control in the VMCS.
7760	*
7761	* @param pVCpu The cross context virtual CPU structure.
7762	*/
7763	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7764	{
7765	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7766	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7767	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7768	AssertRC(rc);
7769	Log4(("Cleared NMI-window exiting\n"));
7770	}
7771
7772
7773	/**
7774	* Evaluates the event to be delivered to the guest and sets it as the pending
7775	* event.
7776	*
7777	* @returns The VT-x guest-interruptibility state.
7778	* @param pVCpu The cross context virtual CPU structure.
7779	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7780	* out-of-sync. Make sure to update the required fields
7781	* before using them.
7782	*/
7783	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7784	{
7785	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7786	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7787	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7788	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7789	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7790
7791	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7792	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7793	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7794	Assert(!TRPMHasTrap(pVCpu));
7795
7796	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7797	APICUpdatePendingInterrupts(pVCpu);
7798
7799	/*
7800	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7801	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7802	*/
7803	/** @todo SMI. SMIs take priority over NMIs. */
7804	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7805	{
7806	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7807	if ( !pVCpu->hm.s.Event.fPending
7808	&& !fBlockNmi
7809	&& !fBlockSti
7810	&& !fBlockMovSS)
7811	{
7812	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7813	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7814	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7815
7816	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7817	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7818	}
7819	else
7820	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7821	}
7822	/*
7823	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7824	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7825	*/
7826	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7827	&& !pVCpu->hm.s.fSingleInstruction)
7828	{
7829	Assert(!DBGFIsStepping(pVCpu));
7830	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7831	AssertRC(rc);
7832	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7833	if ( !pVCpu->hm.s.Event.fPending
7834	&& !fBlockInt
7835	&& !fBlockSti
7836	&& !fBlockMovSS)
7837	{
7838	uint8_t u8Interrupt;
7839	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7840	if (RT_SUCCESS(rc))
7841	{
7842	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7843	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7844	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7845
7846	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7847	}
7848	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7849	{
7850	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7851	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7852	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7853
7854	/*
7855	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7856	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7857	* need to re-set this force-flag here.
7858	*/
7859	}
7860	else
7861	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7862	}
7863	else
7864	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7865	}
7866
7867	return uIntrState;
7868	}
7869
7870
7871	/**
7872	* Sets a pending-debug exception to be delivered to the guest if the guest is
7873	* single-stepping in the VMCS.
7874	*
7875	* @param pVCpu The cross context virtual CPU structure.
7876	*/
7877	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7878	{
7879	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7880	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7881	AssertRC(rc);
7882	}
7883
7884
7885	/**
7886	* Injects any pending events into the guest if the guest is in a state to
7887	* receive them.
7888	*
7889	* @returns Strict VBox status code (i.e. informational status codes too).
7890	* @param pVCpu The cross context virtual CPU structure.
7891	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7892	* out-of-sync. Make sure to update the required fields
7893	* before using them.
7894	* @param uIntrState The VT-x guest-interruptibility state.
7895	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7896	* return VINF_EM_DBG_STEPPED if the event was
7897	* dispatched directly.
7898	*/
7899	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7900	{
7901	HMVMX_ASSERT_PREEMPT_SAFE();
7902	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7903
7904	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7905	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7906
7907	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7908	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7909	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7910	Assert(!TRPMHasTrap(pVCpu));
7911
7912	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7913	if (pVCpu->hm.s.Event.fPending)
7914	{
7915	/*
7916	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7917	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7918	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7919	*
7920	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7921	*/
7922	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7923	#ifdef VBOX_STRICT
7924	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7925	{
7926	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7927	Assert(!fBlockInt);
7928	Assert(!fBlockSti);
7929	Assert(!fBlockMovSS);
7930	}
7931	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7932	{
7933	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7934	Assert(!fBlockSti);
7935	Assert(!fBlockMovSS);
7936	Assert(!fBlockNmi);
7937	}
7938	#endif
7939	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7940	(uint8_t)uIntType));
7941	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7942	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7943	fStepping, &uIntrState);
7944	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7945
7946	/* Update the interruptibility-state as it could have been changed by
7947	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7948	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7949	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7950
7951	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7952	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7953	else
7954	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7955	}
7956
7957	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7958	if ( fBlockSti
7959	\|\| fBlockMovSS)
7960	{
7961	if (!pVCpu->hm.s.fSingleInstruction)
7962	{
7963	/*
7964	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7965	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7966	* See Intel spec. 27.3.4 "Saving Non-Register State".
7967	*/
7968	Assert(!DBGFIsStepping(pVCpu));
7969	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7970	AssertRCReturn(rc2, rc2);
7971	if (pMixedCtx->eflags.Bits.u1TF)
7972	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7973	}
7974	else if (pMixedCtx->eflags.Bits.u1TF)
7975	{
7976	/*
7977	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7978	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7979	*/
7980	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7981	uIntrState = 0;
7982	}
7983	}
7984
7985	/*
7986	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7987	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7988	*/
7989	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7990	AssertRC(rc2);
7991
7992	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7993	NOREF(fBlockMovSS); NOREF(fBlockSti);
7994	return rcStrict;
7995	}
7996
7997
7998	/**
7999	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8000	*
8001	* @param pVCpu The cross context virtual CPU structure.
8002	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8003	* out-of-sync. Make sure to update the required fields
8004	* before using them.
8005	*/
8006	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8007	{
8008	NOREF(pMixedCtx);
8009	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8010	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8011	}
8012
8013
8014	/**
8015	* Injects a double-fault (\#DF) exception into the VM.
8016	*
8017	* @returns Strict VBox status code (i.e. informational status codes too).
8018	* @param pVCpu The cross context virtual CPU structure.
8019	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8020	* out-of-sync. Make sure to update the required fields
8021	* before using them.
8022	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8023	* and should return VINF_EM_DBG_STEPPED if the event
8024	* is injected directly (register modified by us, not
8025	* by hardware on VM-entry).
8026	* @param puIntrState Pointer to the current guest interruptibility-state.
8027	* This interruptibility-state will be updated if
8028	* necessary. This cannot not be NULL.
8029	*/
8030	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8031	{
8032	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8033	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8034	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8035	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8036	fStepping, puIntrState);
8037	}
8038
8039
8040	/**
8041	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8042	*
8043	* @param pVCpu The cross context virtual CPU structure.
8044	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8045	* out-of-sync. Make sure to update the required fields
8046	* before using them.
8047	*/
8048	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8049	{
8050	NOREF(pMixedCtx);
8051	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8052	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8053	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8054	}
8055
8056
8057	/**
8058	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8059	*
8060	* @param pVCpu The cross context virtual CPU structure.
8061	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8062	* out-of-sync. Make sure to update the required fields
8063	* before using them.
8064	* @param cbInstr The value of RIP that is to be pushed on the guest
8065	* stack.
8066	*/
8067	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8068	{
8069	NOREF(pMixedCtx);
8070	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8071	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8072	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8073	}
8074
8075
8076	/**
8077	* Injects a general-protection (\#GP) fault into the VM.
8078	*
8079	* @returns Strict VBox status code (i.e. informational status codes too).
8080	* @param pVCpu The cross context virtual CPU structure.
8081	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8082	* out-of-sync. Make sure to update the required fields
8083	* before using them.
8084	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8085	* mode, i.e. in real-mode it's not valid).
8086	* @param u32ErrorCode The error code associated with the \#GP.
8087	* @param fStepping Whether we're running in
8088	* hmR0VmxRunGuestCodeStep() and should return
8089	* VINF_EM_DBG_STEPPED if the event is injected
8090	* directly (register modified by us, not by
8091	* hardware on VM-entry).
8092	* @param puIntrState Pointer to the current guest interruptibility-state.
8093	* This interruptibility-state will be updated if
8094	* necessary. This cannot not be NULL.
8095	*/
8096	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8097	bool fStepping, uint32_t *puIntrState)
8098	{
8099	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8100	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8101	if (fErrorCodeValid)
8102	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8103	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8104	fStepping, puIntrState);
8105	}
8106
8107
8108	#if 0 /* unused */
8109	/**
8110	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8111	* VM.
8112	*
8113	* @param pVCpu The cross context virtual CPU structure.
8114	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8115	* out-of-sync. Make sure to update the required fields
8116	* before using them.
8117	* @param u32ErrorCode The error code associated with the \#GP.
8118	*/
8119	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8120	{
8121	NOREF(pMixedCtx);
8122	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8123	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8124	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8125	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8126	}
8127	#endif /* unused */
8128
8129
8130	/**
8131	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8132	*
8133	* @param pVCpu The cross context virtual CPU structure.
8134	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8135	* out-of-sync. Make sure to update the required fields
8136	* before using them.
8137	* @param uVector The software interrupt vector number.
8138	* @param cbInstr The value of RIP that is to be pushed on the guest
8139	* stack.
8140	*/
8141	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8142	{
8143	NOREF(pMixedCtx);
8144	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8145	if ( uVector == X86_XCPT_BP
8146	\|\| uVector == X86_XCPT_OF)
8147	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8148	else
8149	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8150	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8151	}
8152
8153
8154	/**
8155	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8156	* stack.
8157	*
8158	* @returns Strict VBox status code (i.e. informational status codes too).
8159	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8160	* @param pVM The cross context VM structure.
8161	* @param pMixedCtx Pointer to the guest-CPU context.
8162	* @param uValue The value to push to the guest stack.
8163	*/
8164	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8165	{
8166	/*
8167	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8168	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8169	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8170	*/
8171	if (pMixedCtx->sp == 1)
8172	return VINF_EM_RESET;
8173	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8174	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8175	AssertRC(rc);
8176	return rc;
8177	}
8178
8179
8180	/**
8181	* Injects an event into the guest upon VM-entry by updating the relevant fields
8182	* in the VM-entry area in the VMCS.
8183	*
8184	* @returns Strict VBox status code (i.e. informational status codes too).
8185	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8186	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8187	*
8188	* @param pVCpu The cross context virtual CPU structure.
8189	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8190	* be out-of-sync. Make sure to update the required
8191	* fields before using them.
8192	* @param u64IntInfo The VM-entry interruption-information field.
8193	* @param cbInstr The VM-entry instruction length in bytes (for
8194	* software interrupts, exceptions and privileged
8195	* software exceptions).
8196	* @param u32ErrCode The VM-entry exception error code.
8197	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8198	* @param puIntrState Pointer to the current guest interruptibility-state.
8199	* This interruptibility-state will be updated if
8200	* necessary. This cannot not be NULL.
8201	* @param fStepping Whether we're running in
8202	* hmR0VmxRunGuestCodeStep() and should return
8203	* VINF_EM_DBG_STEPPED if the event is injected
8204	* directly (register modified by us, not by
8205	* hardware on VM-entry).
8206	*
8207	* @remarks Requires CR0!
8208	*/
8209	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8210	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8211	uint32_t *puIntrState)
8212	{
8213	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8214	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8215	Assert(puIntrState);
8216	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8217
8218	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8219	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8220
8221	#ifdef VBOX_STRICT
8222	/* Validate the error-code-valid bit for hardware exceptions. */
8223	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
8224	{
8225	switch (uVector)
8226	{
8227	case X86_XCPT_PF:
8228	case X86_XCPT_DF:
8229	case X86_XCPT_TS:
8230	case X86_XCPT_NP:
8231	case X86_XCPT_SS:
8232	case X86_XCPT_GP:
8233	case X86_XCPT_AC:
8234	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8235	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8236	/* fall thru */
8237	default:
8238	break;
8239	}
8240	}
8241	#endif
8242
8243	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8244	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8245	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8246
8247	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8248
8249	/* We require CR0 to check if the guest is in real-mode. */
8250	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8251	AssertRCReturn(rc, rc);
8252
8253	/*
8254	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8255	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8256	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8257	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8258	*/
8259	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8260	{
8261	PVM pVM = pVCpu->CTX_SUFF(pVM);
8262	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8263	{
8264	Assert(PDMVmmDevHeapIsEnabled(pVM));
8265	Assert(pVM->hm.s.vmx.pRealModeTSS);
8266
8267	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8268	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8269	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8270	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8271	AssertRCReturn(rc, rc);
8272	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8273
8274	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8275	size_t const cbIdtEntry = sizeof(X86IDTR16);
8276	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8277	{
8278	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8279	if (uVector == X86_XCPT_DF)
8280	return VINF_EM_RESET;
8281
8282	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8283	if (uVector == X86_XCPT_GP)
8284	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8285
8286	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8287	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8288	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8289	fStepping, puIntrState);
8290	}
8291
8292	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8293	uint16_t uGuestIp = pMixedCtx->ip;
8294	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8295	{
8296	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8297	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8298	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8299	}
8300	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8301	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8302
8303	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8304	X86IDTR16 IdtEntry;
8305	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8306	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8307	AssertRCReturn(rc, rc);
8308
8309	/* Construct the stack frame for the interrupt/exception handler. */
8310	VBOXSTRICTRC rcStrict;
8311	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8312	if (rcStrict == VINF_SUCCESS)
8313	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8314	if (rcStrict == VINF_SUCCESS)
8315	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8316
8317	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8318	if (rcStrict == VINF_SUCCESS)
8319	{
8320	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8321	pMixedCtx->rip = IdtEntry.offSel;
8322	pMixedCtx->cs.Sel = IdtEntry.uSel;
8323	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8324	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8325	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8326	&& uVector == X86_XCPT_PF)
8327	pMixedCtx->cr2 = GCPtrFaultAddress;
8328
8329	/* If any other guest-state bits are changed here, make sure to update
8330	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8331	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8332	\| HM_CHANGED_GUEST_RIP
8333	\| HM_CHANGED_GUEST_RFLAGS
8334	\| HM_CHANGED_GUEST_RSP);
8335
8336	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8337	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8338	{
8339	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8340	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8341	Log4(("Clearing inhibition due to STI.\n"));
8342	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8343	}
8344	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8345	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8346
8347	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8348	it, if we are returning to ring-3 before executing guest code. */
8349	pVCpu->hm.s.Event.fPending = false;
8350
8351	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8352	if (fStepping)
8353	rcStrict = VINF_EM_DBG_STEPPED;
8354	}
8355	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8356	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8357	return rcStrict;
8358	}
8359
8360	/*
8361	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8362	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8363	*/
8364	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8365	}
8366
8367	/* Validate. */
8368	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8369	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8370	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8371
8372	/* Inject. */
8373	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8374	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8375	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8376	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8377
8378	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8379	&& uVector == X86_XCPT_PF)
8380	pMixedCtx->cr2 = GCPtrFaultAddress;
8381
8382	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8383	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8384
8385	AssertRCReturn(rc, rc);
8386	return VINF_SUCCESS;
8387	}
8388
8389
8390	/**
8391	* Clears the interrupt-window exiting control in the VMCS and if necessary
8392	* clears the current event in the VMCS as well.
8393	*
8394	* @returns VBox status code.
8395	* @param pVCpu The cross context virtual CPU structure.
8396	*
8397	* @remarks Use this function only to clear events that have not yet been
8398	* delivered to the guest but are injected in the VMCS!
8399	* @remarks No-long-jump zone!!!
8400	*/
8401	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8402	{
8403	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8404
8405	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8406	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8407
8408	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8409	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8410	}
8411
8412
8413	/**
8414	* Enters the VT-x session.
8415	*
8416	* @returns VBox status code.
8417	* @param pVM The cross context VM structure.
8418	* @param pVCpu The cross context virtual CPU structure.
8419	* @param pCpu Pointer to the CPU info struct.
8420	*/
8421	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8422	{
8423	AssertPtr(pVM);
8424	AssertPtr(pVCpu);
8425	Assert(pVM->hm.s.vmx.fSupported);
8426	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8427	NOREF(pCpu); NOREF(pVM);
8428
8429	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8430	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8431
8432	#ifdef VBOX_STRICT
8433	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8434	RTCCUINTREG uHostCR4 = ASMGetCR4();
8435	if (!(uHostCR4 & X86_CR4_VMXE))
8436	{
8437	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8438	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8439	}
8440	#endif
8441
8442	/*
8443	* Load the VCPU's VMCS as the current (and active) one.
8444	*/
8445	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8446	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8447	if (RT_FAILURE(rc))
8448	return rc;
8449
8450	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8451	pVCpu->hm.s.fLeaveDone = false;
8452	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8453
8454	return VINF_SUCCESS;
8455	}
8456
8457
8458	/**
8459	* The thread-context callback (only on platforms which support it).
8460	*
8461	* @param enmEvent The thread-context event.
8462	* @param pVCpu The cross context virtual CPU structure.
8463	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8464	* @thread EMT(pVCpu)
8465	*/
8466	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8467	{
8468	NOREF(fGlobalInit);
8469
8470	switch (enmEvent)
8471	{
8472	case RTTHREADCTXEVENT_OUT:
8473	{
8474	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8475	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8476	VMCPU_ASSERT_EMT(pVCpu);
8477
8478	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8479
8480	/* No longjmps (logger flushes, locks) in this fragile context. */
8481	VMMRZCallRing3Disable(pVCpu);
8482	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8483
8484	/*
8485	* Restore host-state (FPU, debug etc.)
8486	*/
8487	if (!pVCpu->hm.s.fLeaveDone)
8488	{
8489	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8490	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8491	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8492	pVCpu->hm.s.fLeaveDone = true;
8493	}
8494
8495	/* Leave HM context, takes care of local init (term). */
8496	int rc = HMR0LeaveCpu(pVCpu);
8497	AssertRC(rc); NOREF(rc);
8498
8499	/* Restore longjmp state. */
8500	VMMRZCallRing3Enable(pVCpu);
8501	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8502	break;
8503	}
8504
8505	case RTTHREADCTXEVENT_IN:
8506	{
8507	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8508	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8509	VMCPU_ASSERT_EMT(pVCpu);
8510
8511	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8512	VMMRZCallRing3Disable(pVCpu);
8513	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8514
8515	/* Initialize the bare minimum state required for HM. This takes care of
8516	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8517	int rc = HMR0EnterCpu(pVCpu);
8518	AssertRC(rc);
8519	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8520
8521	/* Load the active VMCS as the current one. */
8522	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8523	{
8524	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8525	AssertRC(rc); NOREF(rc);
8526	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8527	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8528	}
8529	pVCpu->hm.s.fLeaveDone = false;
8530
8531	/* Restore longjmp state. */
8532	VMMRZCallRing3Enable(pVCpu);
8533	break;
8534	}
8535
8536	default:
8537	break;
8538	}
8539	}
8540
8541
8542	/**
8543	* Saves the host state in the VMCS host-state.
8544	* Sets up the VM-exit MSR-load area.
8545	*
8546	* The CPU state will be loaded from these fields on every successful VM-exit.
8547	*
8548	* @returns VBox status code.
8549	* @param pVM The cross context VM structure.
8550	* @param pVCpu The cross context virtual CPU structure.
8551	*
8552	* @remarks No-long-jump zone!!!
8553	*/
8554	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8555	{
8556	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8557
8558	int rc = VINF_SUCCESS;
8559	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8560	{
8561	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8562	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8563
8564	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8565	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8566
8567	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8568	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8569
8570	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8571	}
8572	return rc;
8573	}
8574
8575
8576	/**
8577	* Saves the host state in the VMCS host-state.
8578	*
8579	* @returns VBox status code.
8580	* @param pVM The cross context VM structure.
8581	* @param pVCpu The cross context virtual CPU structure.
8582	*
8583	* @remarks No-long-jump zone!!!
8584	*/
8585	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8586	{
8587	AssertPtr(pVM);
8588	AssertPtr(pVCpu);
8589
8590	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8591
8592	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8593	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8594	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8595	return hmR0VmxSaveHostState(pVM, pVCpu);
8596	}
8597
8598
8599	/**
8600	* Loads the guest state into the VMCS guest-state area.
8601	*
8602	* The will typically be done before VM-entry when the guest-CPU state and the
8603	* VMCS state may potentially be out of sync.
8604	*
8605	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8606	* VM-entry controls.
8607	* Sets up the appropriate VMX non-root function to execute guest code based on
8608	* the guest CPU mode.
8609	*
8610	* @returns VBox strict status code.
8611	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8612	* without unrestricted guest access and the VMMDev is not presently
8613	* mapped (e.g. EFI32).
8614	*
8615	* @param pVM The cross context VM structure.
8616	* @param pVCpu The cross context virtual CPU structure.
8617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8618	* out-of-sync. Make sure to update the required fields
8619	* before using them.
8620	*
8621	* @remarks No-long-jump zone!!!
8622	*/
8623	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8624	{
8625	AssertPtr(pVM);
8626	AssertPtr(pVCpu);
8627	AssertPtr(pMixedCtx);
8628	HMVMX_ASSERT_PREEMPT_SAFE();
8629
8630	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8631
8632	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8633
8634	/* Determine real-on-v86 mode. */
8635	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8636	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8637	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8638	{
8639	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8640	}
8641
8642	/*
8643	* Load the guest-state into the VMCS.
8644	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8645	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8646	*/
8647	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8648	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8649
8650	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8651	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8652	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8653
8654	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8655	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8656	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8657
8658	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8659	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8660
8661	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8662	if (rcStrict == VINF_SUCCESS)
8663	{ /* likely */ }
8664	else
8665	{
8666	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8667	return rcStrict;
8668	}
8669
8670	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8671	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8672	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8673
8674	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8675	determine we don't have to swap EFER after all. */
8676	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8677	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8678
8679	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8680	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8681
8682	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8683	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8684
8685	/*
8686	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8687	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8688	*/
8689	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8690	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8691
8692	/* Clear any unused and reserved bits. */
8693	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8694
8695	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8696	return rc;
8697	}
8698
8699
8700	/**
8701	* Loads the state shared between the host and guest into the VMCS.
8702	*
8703	* @param pVM The cross context VM structure.
8704	* @param pVCpu The cross context virtual CPU structure.
8705	* @param pCtx Pointer to the guest-CPU context.
8706	*
8707	* @remarks No-long-jump zone!!!
8708	*/
8709	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8710	{
8711	NOREF(pVM);
8712
8713	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8714	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8715
8716	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8717	{
8718	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8719	AssertRC(rc);
8720	}
8721
8722	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8723	{
8724	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8725	AssertRC(rc);
8726
8727	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8728	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8729	{
8730	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8731	AssertRC(rc);
8732	}
8733	}
8734
8735	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8736	{
8737	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8738	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8739	}
8740
8741	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8742	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8743	{
8744	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8745	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8746	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8747	AssertRC(rc);
8748	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8749	}
8750
8751	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8752	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8753	}
8754
8755
8756	/**
8757	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8758	*
8759	* @returns Strict VBox status code (i.e. informational status codes too).
8760	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8761	* without unrestricted guest access and the VMMDev is not presently
8762	* mapped (e.g. EFI32).
8763	*
8764	* @param pVM The cross context VM structure.
8765	* @param pVCpu The cross context virtual CPU structure.
8766	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8767	* out-of-sync. Make sure to update the required fields
8768	* before using them.
8769	*
8770	* @remarks No-long-jump zone!!!
8771	*/
8772	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8773	{
8774	HMVMX_ASSERT_PREEMPT_SAFE();
8775	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8776	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8777
8778	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8779	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8780	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8781	#endif
8782
8783	/*
8784	* RIP is what changes the most often and hence if it's the only bit needing to be
8785	* updated, we shall handle it early for performance reasons.
8786	*/
8787	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8788	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8789	{
8790	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8791	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8792	{ /* likely */}
8793	else
8794	{
8795	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8796	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8797	}
8798	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8799	}
8800	else if (HMCPU_CF_VALUE(pVCpu))
8801	{
8802	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8803	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8804	{ /* likely */}
8805	else
8806	{
8807	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8808	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8809	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8810	return rcStrict;
8811	}
8812	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8813	}
8814
8815	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8816	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8817	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8818	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8819	return rcStrict;
8820	}
8821
8822
8823	/**
8824	* Does the preparations before executing guest code in VT-x.
8825	*
8826	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8827	* recompiler/IEM. We must be cautious what we do here regarding committing
8828	* guest-state information into the VMCS assuming we assuredly execute the
8829	* guest in VT-x mode.
8830	*
8831	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8832	* the common-state (TRPM/forceflags), we must undo those changes so that the
8833	* recompiler/IEM can (and should) use them when it resumes guest execution.
8834	* Otherwise such operations must be done when we can no longer exit to ring-3.
8835	*
8836	* @returns Strict VBox status code (i.e. informational status codes too).
8837	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8838	* have been disabled.
8839	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8840	* double-fault into the guest.
8841	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8842	* dispatched directly.
8843	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8844	*
8845	* @param pVM The cross context VM structure.
8846	* @param pVCpu The cross context virtual CPU structure.
8847	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8848	* out-of-sync. Make sure to update the required fields
8849	* before using them.
8850	* @param pVmxTransient Pointer to the VMX transient structure.
8851	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8852	* us ignore some of the reasons for returning to
8853	* ring-3, and return VINF_EM_DBG_STEPPED if event
8854	* dispatching took place.
8855	*/
8856	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8857	{
8858	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8859
8860	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8861	PGMRZDynMapFlushAutoSet(pVCpu);
8862	#endif
8863
8864	/* Check force flag actions that might require us to go back to ring-3. */
8865	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8866	if (rcStrict == VINF_SUCCESS)
8867	{ /* FFs doesn't get set all the time. */ }
8868	else
8869	return rcStrict;
8870
8871	#ifndef IEM_VERIFICATION_MODE_FULL
8872	/*
8873	* Setup the virtualized-APIC accesses.
8874	*
8875	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8876	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8877	*
8878	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
8879	*/
8880	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8881	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8882	&& PDMHasApic(pVM))
8883	{
8884	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8885	Assert(u64MsrApicBase);
8886	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8887
8888	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8889
8890	/* Unalias any existing mapping. */
8891	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8892	AssertRCReturn(rc, rc);
8893
8894	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8895	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8896	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8897	AssertRCReturn(rc, rc);
8898
8899	/* Update the per-VCPU cache of the APIC base MSR. */
8900	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8901	}
8902	#endif /* !IEM_VERIFICATION_MODE_FULL */
8903
8904	if (TRPMHasTrap(pVCpu))
8905	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8906	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8907
8908	/*
8909	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8910	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8911	*/
8912	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8913	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8914	{ /* likely */ }
8915	else
8916	{
8917	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8918	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8919	return rcStrict;
8920	}
8921
8922	/*
8923	* No longjmps to ring-3 from this point on!!!
8924	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8925	* This also disables flushing of the R0-logger instance (if any).
8926	*/
8927	VMMRZCallRing3Disable(pVCpu);
8928
8929	/*
8930	* Load the guest state bits.
8931	*
8932	* We cannot perform longjmps while loading the guest state because we do not preserve the
8933	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8934	* CPU migration.
8935	*
8936	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8937	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8938	* Hence, loading of the guest state needs to be done -after- injection of events.
8939	*/
8940	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8941	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8942	{ /* likely */ }
8943	else
8944	{
8945	VMMRZCallRing3Enable(pVCpu);
8946	return rcStrict;
8947	}
8948
8949	/*
8950	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8951	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8952	*
8953	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8954	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8955	*
8956	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8957	* executing guest code.
8958	*/
8959	pVmxTransient->fEFlags = ASMIntDisableFlags();
8960
8961	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8962	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8963	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8964	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8965	{
8966	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8967	{
8968	pVCpu->hm.s.Event.fPending = false;
8969
8970	/*
8971	* We've injected any pending events. This is really the point of no return (to ring-3).
8972	*
8973	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
8974	* returns from this function, so don't enable them here.
8975	*/
8976	return VINF_SUCCESS;
8977	}
8978
8979	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8980	rcStrict = VINF_EM_RAW_INTERRUPT;
8981	}
8982	else
8983	{
8984	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8985	rcStrict = VINF_EM_RAW_TO_R3;
8986	}
8987
8988	ASMSetFlags(pVmxTransient->fEFlags);
8989	VMMRZCallRing3Enable(pVCpu);
8990
8991	return rcStrict;
8992	}
8993
8994
8995	/**
8996	* Prepares to run guest code in VT-x and we've committed to doing so. This
8997	* means there is no backing out to ring-3 or anywhere else at this
8998	* point.
8999	*
9000	* @param pVM The cross context VM structure.
9001	* @param pVCpu The cross context virtual CPU structure.
9002	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9003	* out-of-sync. Make sure to update the required fields
9004	* before using them.
9005	* @param pVmxTransient Pointer to the VMX transient structure.
9006	*
9007	* @remarks Called with preemption disabled.
9008	* @remarks No-long-jump zone!!!
9009	*/
9010	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9011	{
9012	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9013	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9014	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9015
9016	/*
9017	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9018	*/
9019	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9020	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9021
9022	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9023	if (!CPUMIsGuestFPUStateActive(pVCpu))
9024	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9025	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9026	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9027	#endif
9028
9029	if ( pVCpu->hm.s.fPreloadGuestFpu
9030	&& !CPUMIsGuestFPUStateActive(pVCpu))
9031	{
9032	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9033	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9034	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9035	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9036	}
9037
9038	/*
9039	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9040	*/
9041	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9042	&& pVCpu->hm.s.vmx.cMsrs > 0)
9043	{
9044	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9045	}
9046
9047	/*
9048	* Load the host state bits as we may've been preempted (only happens when
9049	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9050	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9051	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9052	* See @bugref{8432}.
9053	*/
9054	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9055	{
9056	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9057	AssertRC(rc);
9058	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9059	}
9060	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9061
9062	/*
9063	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9064	*/
9065	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9066	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9067	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9068
9069	/* Store status of the shared guest-host state at the time of VM-entry. */
9070	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9071	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9072	{
9073	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9074	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9075	}
9076	else
9077	#endif
9078	{
9079	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9080	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9081	}
9082	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9083
9084	/*
9085	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9086	*/
9087	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9088	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9089
9090	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9091	RTCPUID idCurrentCpu = pCpu->idCpu;
9092	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9093	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9094	{
9095	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9096	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9097	}
9098
9099	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9100	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9101	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9102	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9103
9104	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9105
9106	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9107	to start executing. */
9108
9109	/*
9110	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9111	*/
9112	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9113	{
9114	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9115	{
9116	bool fMsrUpdated;
9117	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9118	AssertRC(rc2);
9119	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9120
9121	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9122	&fMsrUpdated);
9123	AssertRC(rc2);
9124	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9125
9126	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9127	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9128	}
9129	else
9130	{
9131	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9132	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9133	}
9134	}
9135
9136	#ifdef VBOX_STRICT
9137	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9138	hmR0VmxCheckHostEferMsr(pVCpu);
9139	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9140	#endif
9141	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9142	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9143	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9144	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9145	#endif
9146	}
9147
9148
9149	/**
9150	* Performs some essential restoration of state after running guest code in
9151	* VT-x.
9152	*
9153	* @param pVM The cross context VM structure.
9154	* @param pVCpu The cross context virtual CPU structure.
9155	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9156	* out-of-sync. Make sure to update the required fields
9157	* before using them.
9158	* @param pVmxTransient Pointer to the VMX transient structure.
9159	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9160	*
9161	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9162	*
9163	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9164	* unconditionally when it is safe to do so.
9165	*/
9166	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9167	{
9168	NOREF(pVM);
9169
9170	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9171
9172	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9173	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9174	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9175	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9176	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9177	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9178
9179	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9180	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9181
9182	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9183	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9184	Assert(!ASMIntAreEnabled());
9185	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9186
9187	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9188	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9189	{
9190	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9191	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9192	}
9193	#endif
9194
9195	#if HC_ARCH_BITS == 64
9196	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9197	#endif
9198	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9199	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9200	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9201	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9202	#else
9203	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9204	#endif
9205	#ifdef VBOX_STRICT
9206	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9207	#endif
9208	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9209	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
9210
9211	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9212	uint32_t uExitReason;
9213	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9214	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9215	AssertRC(rc);
9216	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9217	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9218
9219	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
9220	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
9221	{
9222	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9223	pVmxTransient->fVMEntryFailed));
9224	return;
9225	}
9226
9227	/*
9228	* Update the VM-exit history array here even if the VM-entry failed due to:
9229	* - Invalid guest state.
9230	* - MSR loading.
9231	* - Machine-check event.
9232	*
9233	* In any of the above cases we will still have a "valid" VM-exit reason
9234	* despite @a fVMEntryFailed being false.
9235	*
9236	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9237	*/
9238	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9239
9240	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
9241	{
9242	/** @todo We can optimize this by only syncing with our force-flags when
9243	* really needed and keeping the VMCS state as it is for most
9244	* VM-exits. */
9245	/* Update the guest interruptibility-state from the VMCS. */
9246	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9247
9248	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9249	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9250	AssertRC(rc);
9251	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9252	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9253	AssertRC(rc);
9254	#endif
9255
9256	/*
9257	* Sync the TPR shadow with our APIC state.
9258	*/
9259	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9260	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9261	{
9262	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9263	AssertRC(rc);
9264	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9265	}
9266	}
9267	}
9268
9269
9270	/**
9271	* Runs the guest code using VT-x the normal way.
9272	*
9273	* @returns VBox status code.
9274	* @param pVM The cross context VM structure.
9275	* @param pVCpu The cross context virtual CPU structure.
9276	* @param pCtx Pointer to the guest-CPU context.
9277	*
9278	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9279	*/
9280	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9281	{
9282	VMXTRANSIENT VmxTransient;
9283	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9284	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9285	uint32_t cLoops = 0;
9286
9287	for (;; cLoops++)
9288	{
9289	Assert(!HMR0SuspendPending());
9290	HMVMX_ASSERT_CPU_SAFE();
9291
9292	/* Preparatory work for running guest code, this may force us to return
9293	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9294	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9295	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9296	if (rcStrict != VINF_SUCCESS)
9297	break;
9298
9299	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9300	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9301	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9302
9303	/* Restore any residual host-state and save any bits shared between host
9304	and guest into the guest-CPU state. Re-enables interrupts! */
9305	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9306
9307	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9308	if (RT_SUCCESS(rcRun))
9309	{ /* very likely */ }
9310	else
9311	{
9312	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9313	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9314	return rcRun;
9315	}
9316
9317	/* Profile the VM-exit. */
9318	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9319	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9320	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9321	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9322	HMVMX_START_EXIT_DISPATCH_PROF();
9323
9324	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9325
9326	/* Handle the VM-exit. */
9327	#ifdef HMVMX_USE_FUNCTION_TABLE
9328	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9329	#else
9330	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9331	#endif
9332	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9333	if (rcStrict == VINF_SUCCESS)
9334	{
9335	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9336	continue; /* likely */
9337	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9338	rcStrict = VINF_EM_RAW_INTERRUPT;
9339	}
9340	break;
9341	}
9342
9343	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9344	return rcStrict;
9345	}
9346
9347
9348
9349	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9350	* probes.
9351	*
9352	* The following few functions and associated structure contains the bloat
9353	* necessary for providing detailed debug events and dtrace probes as well as
9354	* reliable host side single stepping. This works on the principle of
9355	* "subclassing" the normal execution loop and workers. We replace the loop
9356	* method completely and override selected helpers to add necessary adjustments
9357	* to their core operation.
9358	*
9359	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9360	* any performance for debug and analysis features.
9361	*
9362	* @{
9363	*/
9364
9365	/**
9366	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9367	* the debug run loop.
9368	*/
9369	typedef struct VMXRUNDBGSTATE
9370	{
9371	/** The RIP we started executing at. This is for detecting that we stepped. */
9372	uint64_t uRipStart;
9373	/** The CS we started executing with. */
9374	uint16_t uCsStart;
9375
9376	/** Whether we've actually modified the 1st execution control field. */
9377	bool fModifiedProcCtls : 1;
9378	/** Whether we've actually modified the 2nd execution control field. */
9379	bool fModifiedProcCtls2 : 1;
9380	/** Whether we've actually modified the exception bitmap. */
9381	bool fModifiedXcptBitmap : 1;
9382
9383	/** We desire the modified the CR0 mask to be cleared. */
9384	bool fClearCr0Mask : 1;
9385	/** We desire the modified the CR4 mask to be cleared. */
9386	bool fClearCr4Mask : 1;
9387	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9388	uint32_t fCpe1Extra;
9389	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9390	uint32_t fCpe1Unwanted;
9391	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9392	uint32_t fCpe2Extra;
9393	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9394	uint32_t bmXcptExtra;
9395	/** The sequence number of the Dtrace provider settings the state was
9396	* configured against. */
9397	uint32_t uDtraceSettingsSeqNo;
9398	/** VM-exits to check (one bit per VM-exit). */
9399	uint32_t bmExitsToCheck[3];
9400
9401	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9402	uint32_t fProcCtlsInitial;
9403	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9404	uint32_t fProcCtls2Initial;
9405	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9406	uint32_t bmXcptInitial;
9407	} VMXRUNDBGSTATE;
9408	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9409	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9410
9411
9412	/**
9413	* Initializes the VMXRUNDBGSTATE structure.
9414	*
9415	* @param pVCpu The cross context virtual CPU structure of the
9416	* calling EMT.
9417	* @param pCtx The CPU register context to go with @a pVCpu.
9418	* @param pDbgState The structure to initialize.
9419	*/
9420	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9421	{
9422	pDbgState->uRipStart = pCtx->rip;
9423	pDbgState->uCsStart = pCtx->cs.Sel;
9424
9425	pDbgState->fModifiedProcCtls = false;
9426	pDbgState->fModifiedProcCtls2 = false;
9427	pDbgState->fModifiedXcptBitmap = false;
9428	pDbgState->fClearCr0Mask = false;
9429	pDbgState->fClearCr4Mask = false;
9430	pDbgState->fCpe1Extra = 0;
9431	pDbgState->fCpe1Unwanted = 0;
9432	pDbgState->fCpe2Extra = 0;
9433	pDbgState->bmXcptExtra = 0;
9434	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9435	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9436	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9437	}
9438
9439
9440	/**
9441	* Updates the VMSC fields with changes requested by @a pDbgState.
9442	*
9443	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9444	* immediately before executing guest code, i.e. when interrupts are disabled.
9445	* We don't check status codes here as we cannot easily assert or return in the
9446	* latter case.
9447	*
9448	* @param pVCpu The cross context virtual CPU structure.
9449	* @param pDbgState The debug state.
9450	*/
9451	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9452	{
9453	/*
9454	* Ensure desired flags in VMCS control fields are set.
9455	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9456	*
9457	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9458	* there should be no stale data in pCtx at this point.
9459	*/
9460	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9461	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9462	{
9463	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9464	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9465	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9466	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9467	pDbgState->fModifiedProcCtls = true;
9468	}
9469
9470	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9471	{
9472	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9473	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9474	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9475	pDbgState->fModifiedProcCtls2 = true;
9476	}
9477
9478	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9479	{
9480	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9481	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9482	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9483	pDbgState->fModifiedXcptBitmap = true;
9484	}
9485
9486	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9487	{
9488	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9489	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9490	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9491	}
9492
9493	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9494	{
9495	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9496	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9497	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9498	}
9499	}
9500
9501
9502	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9503	{
9504	/*
9505	* Restore VM-exit control settings as we may not reenter this function the
9506	* next time around.
9507	*/
9508	/* We reload the initial value, trigger what we can of recalculations the
9509	next time around. From the looks of things, that's all that's required atm. */
9510	if (pDbgState->fModifiedProcCtls)
9511	{
9512	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9513	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9514	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9515	AssertRCReturn(rc2, rc2);
9516	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9517	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_DEBUG);
9518	}
9519
9520	/* We're currently the only ones messing with this one, so just restore the
9521	cached value and reload the field. */
9522	if ( pDbgState->fModifiedProcCtls2
9523	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9524	{
9525	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9526	AssertRCReturn(rc2, rc2);
9527	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9528	}
9529
9530	/* If we've modified the exception bitmap, we restore it and trigger
9531	reloading and partial recalculation the next time around. */
9532	if (pDbgState->fModifiedXcptBitmap)
9533	{
9534	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9535	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS \| HM_CHANGED_GUEST_CR0);
9536	}
9537
9538	/* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9539	if (pDbgState->fClearCr0Mask)
9540	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9541
9542	/* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9543	if (pDbgState->fClearCr4Mask)
9544	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9545
9546	return rcStrict;
9547	}
9548
9549
9550	/**
9551	* Configures VM-exit controls for current DBGF and DTrace settings.
9552	*
9553	* This updates @a pDbgState and the VMCS execution control fields to reflect
9554	* the necessary VM-exits demanded by DBGF and DTrace.
9555	*
9556	* @param pVM The cross context VM structure.
9557	* @param pVCpu The cross context virtual CPU structure.
9558	* @param pCtx Pointer to the guest-CPU context.
9559	* @param pDbgState The debug state.
9560	* @param pVmxTransient Pointer to the VMX transient structure. May update
9561	* fUpdateTscOffsettingAndPreemptTimer.
9562	*/
9563	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9564	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9565	{
9566	/*
9567	* Take down the dtrace serial number so we can spot changes.
9568	*/
9569	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9570	ASMCompilerBarrier();
9571
9572	/*
9573	* We'll rebuild most of the middle block of data members (holding the
9574	* current settings) as we go along here, so start by clearing it all.
9575	*/
9576	pDbgState->bmXcptExtra = 0;
9577	pDbgState->fCpe1Extra = 0;
9578	pDbgState->fCpe1Unwanted = 0;
9579	pDbgState->fCpe2Extra = 0;
9580	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9581	pDbgState->bmExitsToCheck[i] = 0;
9582
9583	/*
9584	* Software interrupts (INT XXh) - no idea how to trigger these...
9585	*/
9586	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9587	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9588	{
9589	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9590	}
9591
9592	/*
9593	* INT3 breakpoints - triggered by #BP exceptions.
9594	*/
9595	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9596	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9597
9598	/*
9599	* Exception bitmap and XCPT events+probes.
9600	*/
9601	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9602	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9603	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9604
9605	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9606	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9607	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9608	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9609	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9610	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9611	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9612	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9613	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9614	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9615	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9616	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9617	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9618	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9619	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9620	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9621	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9622	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9623
9624	if (pDbgState->bmXcptExtra)
9625	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9626
9627	/*
9628	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9629	*
9630	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9631	* So, when adding/changing/removing please don't forget to update it.
9632	*
9633	* Some of the macros are picking up local variables to save horizontal space,
9634	* (being able to see it in a table is the lesser evil here).
9635	*/
9636	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9637	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9638	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9639	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9640	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9641	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9642	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9643	} else do { } while (0)
9644	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9645	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9646	{ \
9647	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9648	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9649	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9650	} else do { } while (0)
9651	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9652	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9653	{ \
9654	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9655	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9656	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9657	} else do { } while (0)
9658	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9659	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9660	{ \
9661	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9662	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9663	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9664	} else do { } while (0)
9665
9666	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9667	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9668	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9669	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9670	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9671
9672	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9673	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9674	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9675	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9676	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9677	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9678	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9679	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9680	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9681	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9682	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9683	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9684	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9685	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9686	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9687	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9688	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9689	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9690	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9691	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9692	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9693	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9694	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9695	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9696	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9697	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9698	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9699	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9700	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9701	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9702	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9703	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9704	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9705	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9706	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9707	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9708
9709	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9710	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9711	{
9712	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9713	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9714	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9715	AssertRC(rc2);
9716
9717	#if 0 /** @todo fix me */
9718	pDbgState->fClearCr0Mask = true;
9719	pDbgState->fClearCr4Mask = true;
9720	#endif
9721	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9722	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9723	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9724	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9725	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9726	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9727	require clearing here and in the loop if we start using it. */
9728	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9729	}
9730	else
9731	{
9732	if (pDbgState->fClearCr0Mask)
9733	{
9734	pDbgState->fClearCr0Mask = false;
9735	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9736	}
9737	if (pDbgState->fClearCr4Mask)
9738	{
9739	pDbgState->fClearCr4Mask = false;
9740	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9741	}
9742	}
9743	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9744	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9745
9746	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9747	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9748	{
9749	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9750	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9751	}
9752	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9753	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9754
9755	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9756	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9757	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9758	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9759	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9761	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9762	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9763	#if 0 /** @todo too slow, fix handler. */
9764	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9765	#endif
9766	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9767
9768	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9769	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9770	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9771	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9772	{
9773	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9774	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9775	}
9776	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9777	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9778	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9779	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9780
9781	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9782	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9783	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9784	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9785	{
9786	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9787	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9788	}
9789	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9790	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9791	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9792	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9793
9794	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9795	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9796	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9797	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9798	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9799	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9800	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9801	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9802	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9803	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9804	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9805	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9806	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9807	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9808	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9809	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9810	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9811	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9812	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9813	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9814	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9815	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9816
9817	#undef IS_EITHER_ENABLED
9818	#undef SET_ONLY_XBM_IF_EITHER_EN
9819	#undef SET_CPE1_XBM_IF_EITHER_EN
9820	#undef SET_CPEU_XBM_IF_EITHER_EN
9821	#undef SET_CPE2_XBM_IF_EITHER_EN
9822
9823	/*
9824	* Sanitize the control stuff.
9825	*/
9826	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9827	if (pDbgState->fCpe2Extra)
9828	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9829	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9830	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9831	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9832	{
9833	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9834	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9835	}
9836
9837	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9838	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9839	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9840	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9841	}
9842
9843
9844	/**
9845	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9846	* appropriate.
9847	*
9848	* The caller has checked the VM-exit against the
9849	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9850	* already, so we don't have to do that either.
9851	*
9852	* @returns Strict VBox status code (i.e. informational status codes too).
9853	* @param pVM The cross context VM structure.
9854	* @param pVCpu The cross context virtual CPU structure.
9855	* @param pMixedCtx Pointer to the guest-CPU context.
9856	* @param pVmxTransient Pointer to the VMX-transient structure.
9857	* @param uExitReason The VM-exit reason.
9858	*
9859	* @remarks The name of this function is displayed by dtrace, so keep it short
9860	* and to the point. No longer than 33 chars long, please.
9861	*/
9862	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9863	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9864	{
9865	/*
9866	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9867	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9868	*
9869	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9870	* does. Must add/change/remove both places. Same ordering, please.
9871	*
9872	* Added/removed events must also be reflected in the next section
9873	* where we dispatch dtrace events.
9874	*/
9875	bool fDtrace1 = false;
9876	bool fDtrace2 = false;
9877	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9878	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9879	uint32_t uEventArg = 0;
9880	#define SET_EXIT(a_EventSubName) \
9881	do { \
9882	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9883	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9884	} while (0)
9885	#define SET_BOTH(a_EventSubName) \
9886	do { \
9887	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9888	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9889	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9890	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9891	} while (0)
9892	switch (uExitReason)
9893	{
9894	case VMX_EXIT_MTF:
9895	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9896
9897	case VMX_EXIT_XCPT_OR_NMI:
9898	{
9899	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9900	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9901	{
9902	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9903	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9904	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9905	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9906	{
9907	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9908	{
9909	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9910	uEventArg = pVmxTransient->uExitIntErrorCode;
9911	}
9912	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9913	switch (enmEvent1)
9914	{
9915	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9916	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9917	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9918	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9919	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9920	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9921	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9922	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9923	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9924	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9925	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9926	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9927	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9928	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9929	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9930	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9931	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9932	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9933	default: break;
9934	}
9935	}
9936	else
9937	AssertFailed();
9938	break;
9939
9940	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9941	uEventArg = idxVector;
9942	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9943	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9944	break;
9945	}
9946	break;
9947	}
9948
9949	case VMX_EXIT_TRIPLE_FAULT:
9950	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9951	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9952	break;
9953	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9954	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9955	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9956	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9957	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9958
9959	/* Instruction specific VM-exits: */
9960	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9961	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9962	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9963	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9964	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9965	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9966	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9967	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9968	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9969	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9970	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9971	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9972	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9973	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9974	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9975	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9976	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9977	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9978	case VMX_EXIT_MOV_CRX:
9979	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9980	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9981	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9982	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9983	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9984	SET_BOTH(CRX_READ);
9985	else
9986	SET_BOTH(CRX_WRITE);
9987	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9988	break;
9989	case VMX_EXIT_MOV_DRX:
9990	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9991	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9992	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9993	SET_BOTH(DRX_READ);
9994	else
9995	SET_BOTH(DRX_WRITE);
9996	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9997	break;
9998	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9999	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10000	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10001	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10002	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10003	case VMX_EXIT_XDTR_ACCESS:
10004	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10005	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10006	{
10007	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10008	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10009	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10010	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10011	}
10012	break;
10013
10014	case VMX_EXIT_TR_ACCESS:
10015	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10016	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10017	{
10018	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10019	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10020	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10021	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10022	}
10023	break;
10024
10025	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10026	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10027	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10028	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10029	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10030	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10031	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10032	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10033	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10034	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10035	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10036
10037	/* Events that aren't relevant at this point. */
10038	case VMX_EXIT_EXT_INT:
10039	case VMX_EXIT_INT_WINDOW:
10040	case VMX_EXIT_NMI_WINDOW:
10041	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10042	case VMX_EXIT_PREEMPT_TIMER:
10043	case VMX_EXIT_IO_INSTR:
10044	break;
10045
10046	/* Errors and unexpected events. */
10047	case VMX_EXIT_INIT_SIGNAL:
10048	case VMX_EXIT_SIPI:
10049	case VMX_EXIT_IO_SMI:
10050	case VMX_EXIT_SMI:
10051	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10052	case VMX_EXIT_ERR_MSR_LOAD:
10053	case VMX_EXIT_ERR_MACHINE_CHECK:
10054	break;
10055
10056	default:
10057	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10058	break;
10059	}
10060	#undef SET_BOTH
10061	#undef SET_EXIT
10062
10063	/*
10064	* Dtrace tracepoints go first. We do them here at once so we don't
10065	* have to copy the guest state saving and stuff a few dozen times.
10066	* Down side is that we've got to repeat the switch, though this time
10067	* we use enmEvent since the probes are a subset of what DBGF does.
10068	*/
10069	if (fDtrace1 \|\| fDtrace2)
10070	{
10071	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10072	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10073	switch (enmEvent1)
10074	{
10075	/** @todo consider which extra parameters would be helpful for each probe. */
10076	case DBGFEVENT_END: break;
10077	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10078	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10079	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10080	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10081	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10082	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10083	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10084	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10085	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10086	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10087	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10088	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10089	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10090	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10091	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10092	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10093	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10094	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10095	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10096	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10097	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10098	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10099	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10100	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10101	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10102	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10103	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10104	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10105	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10106	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10107	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10108	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10109	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10110	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10111	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10112	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10113	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10114	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10115	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10116	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10117	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10118	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10119	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10120	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10121	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10122	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10123	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10124	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10125	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10126	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10127	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10128	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10129	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10130	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10131	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10132	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10133	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10134	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10135	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10136	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10137	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10138	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10139	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10140	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10141	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10142	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10143	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10144	}
10145	switch (enmEvent2)
10146	{
10147	/** @todo consider which extra parameters would be helpful for each probe. */
10148	case DBGFEVENT_END: break;
10149	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10150	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10151	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10152	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10153	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10154	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10155	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10156	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10157	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10158	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10159	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10160	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10161	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10162	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10163	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10164	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10165	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10166	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10167	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10168	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10169	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10170	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10171	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10172	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10173	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10174	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10175	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10176	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10177	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10178	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10179	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10180	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10181	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10182	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10183	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10184	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10185	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10186	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10187	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10188	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10189	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10190	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10191	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10192	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10193	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10194	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10195	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10196	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10197	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10198	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10199	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10200	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10201	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10202	}
10203	}
10204
10205	/*
10206	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10207	* the DBGF call will do a full check).
10208	*
10209	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10210	* Note! If we have to events, we prioritize the first, i.e. the instruction
10211	* one, in order to avoid event nesting.
10212	*/
10213	if ( enmEvent1 != DBGFEVENT_END
10214	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10215	{
10216	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10217	if (rcStrict != VINF_SUCCESS)
10218	return rcStrict;
10219	}
10220	else if ( enmEvent2 != DBGFEVENT_END
10221	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10222	{
10223	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10224	if (rcStrict != VINF_SUCCESS)
10225	return rcStrict;
10226	}
10227
10228	return VINF_SUCCESS;
10229	}
10230
10231
10232	/**
10233	* Single-stepping VM-exit filtering.
10234	*
10235	* This is preprocessing the VM-exits and deciding whether we've gotten far
10236	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10237	* handling is performed.
10238	*
10239	* @returns Strict VBox status code (i.e. informational status codes too).
10240	* @param pVM The cross context VM structure.
10241	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10242	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10243	* out-of-sync. Make sure to update the required
10244	* fields before using them.
10245	* @param pVmxTransient Pointer to the VMX-transient structure.
10246	* @param uExitReason The VM-exit reason.
10247	* @param pDbgState The debug state.
10248	*/
10249	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10250	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10251	{
10252	/*
10253	* Expensive (saves context) generic dtrace VM-exit probe.
10254	*/
10255	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10256	{ /* more likely */ }
10257	else
10258	{
10259	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10260	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10261	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10262	}
10263
10264	/*
10265	* Check for host NMI, just to get that out of the way.
10266	*/
10267	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10268	{ /* normally likely */ }
10269	else
10270	{
10271	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10272	AssertRCReturn(rc2, rc2);
10273	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10274	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10275	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10276	}
10277
10278	/*
10279	* Check for single stepping event if we're stepping.
10280	*/
10281	if (pVCpu->hm.s.fSingleInstruction)
10282	{
10283	switch (uExitReason)
10284	{
10285	case VMX_EXIT_MTF:
10286	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10287
10288	/* Various events: */
10289	case VMX_EXIT_XCPT_OR_NMI:
10290	case VMX_EXIT_EXT_INT:
10291	case VMX_EXIT_TRIPLE_FAULT:
10292	case VMX_EXIT_INT_WINDOW:
10293	case VMX_EXIT_NMI_WINDOW:
10294	case VMX_EXIT_TASK_SWITCH:
10295	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10296	case VMX_EXIT_APIC_ACCESS:
10297	case VMX_EXIT_EPT_VIOLATION:
10298	case VMX_EXIT_EPT_MISCONFIG:
10299	case VMX_EXIT_PREEMPT_TIMER:
10300
10301	/* Instruction specific VM-exits: */
10302	case VMX_EXIT_CPUID:
10303	case VMX_EXIT_GETSEC:
10304	case VMX_EXIT_HLT:
10305	case VMX_EXIT_INVD:
10306	case VMX_EXIT_INVLPG:
10307	case VMX_EXIT_RDPMC:
10308	case VMX_EXIT_RDTSC:
10309	case VMX_EXIT_RSM:
10310	case VMX_EXIT_VMCALL:
10311	case VMX_EXIT_VMCLEAR:
10312	case VMX_EXIT_VMLAUNCH:
10313	case VMX_EXIT_VMPTRLD:
10314	case VMX_EXIT_VMPTRST:
10315	case VMX_EXIT_VMREAD:
10316	case VMX_EXIT_VMRESUME:
10317	case VMX_EXIT_VMWRITE:
10318	case VMX_EXIT_VMXOFF:
10319	case VMX_EXIT_VMXON:
10320	case VMX_EXIT_MOV_CRX:
10321	case VMX_EXIT_MOV_DRX:
10322	case VMX_EXIT_IO_INSTR:
10323	case VMX_EXIT_RDMSR:
10324	case VMX_EXIT_WRMSR:
10325	case VMX_EXIT_MWAIT:
10326	case VMX_EXIT_MONITOR:
10327	case VMX_EXIT_PAUSE:
10328	case VMX_EXIT_XDTR_ACCESS:
10329	case VMX_EXIT_TR_ACCESS:
10330	case VMX_EXIT_INVEPT:
10331	case VMX_EXIT_RDTSCP:
10332	case VMX_EXIT_INVVPID:
10333	case VMX_EXIT_WBINVD:
10334	case VMX_EXIT_XSETBV:
10335	case VMX_EXIT_RDRAND:
10336	case VMX_EXIT_INVPCID:
10337	case VMX_EXIT_VMFUNC:
10338	case VMX_EXIT_RDSEED:
10339	case VMX_EXIT_XSAVES:
10340	case VMX_EXIT_XRSTORS:
10341	{
10342	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10343	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10344	AssertRCReturn(rc2, rc2);
10345	if ( pMixedCtx->rip != pDbgState->uRipStart
10346	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10347	return VINF_EM_DBG_STEPPED;
10348	break;
10349	}
10350
10351	/* Errors and unexpected events: */
10352	case VMX_EXIT_INIT_SIGNAL:
10353	case VMX_EXIT_SIPI:
10354	case VMX_EXIT_IO_SMI:
10355	case VMX_EXIT_SMI:
10356	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10357	case VMX_EXIT_ERR_MSR_LOAD:
10358	case VMX_EXIT_ERR_MACHINE_CHECK:
10359	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10360	break;
10361
10362	default:
10363	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10364	break;
10365	}
10366	}
10367
10368	/*
10369	* Check for debugger event breakpoints and dtrace probes.
10370	*/
10371	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10372	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10373	{
10374	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10375	if (rcStrict != VINF_SUCCESS)
10376	return rcStrict;
10377	}
10378
10379	/*
10380	* Normal processing.
10381	*/
10382	#ifdef HMVMX_USE_FUNCTION_TABLE
10383	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10384	#else
10385	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10386	#endif
10387	}
10388
10389
10390	/**
10391	* Single steps guest code using VT-x.
10392	*
10393	* @returns Strict VBox status code (i.e. informational status codes too).
10394	* @param pVM The cross context VM structure.
10395	* @param pVCpu The cross context virtual CPU structure.
10396	* @param pCtx Pointer to the guest-CPU context.
10397	*
10398	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10399	*/
10400	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10401	{
10402	VMXTRANSIENT VmxTransient;
10403	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10404
10405	/* Set HMCPU indicators. */
10406	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10407	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10408	pVCpu->hm.s.fDebugWantRdTscExit = false;
10409	pVCpu->hm.s.fUsingDebugLoop = true;
10410
10411	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10412	VMXRUNDBGSTATE DbgState;
10413	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10414	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10415
10416	/*
10417	* The loop.
10418	*/
10419	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10420	for (uint32_t cLoops = 0; ; cLoops++)
10421	{
10422	Assert(!HMR0SuspendPending());
10423	HMVMX_ASSERT_CPU_SAFE();
10424	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10425
10426	/*
10427	* Preparatory work for running guest code, this may force us to return
10428	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10429	*/
10430	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10431	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10432	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10433	if (rcStrict != VINF_SUCCESS)
10434	break;
10435
10436	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10437	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10438
10439	/*
10440	* Now we can run the guest code.
10441	*/
10442	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10443
10444	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10445
10446	/*
10447	* Restore any residual host-state and save any bits shared between host
10448	* and guest into the guest-CPU state. Re-enables interrupts!
10449	*/
10450	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10451
10452	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10453	if (RT_SUCCESS(rcRun))
10454	{ /* very likely */ }
10455	else
10456	{
10457	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10458	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10459	return rcRun;
10460	}
10461
10462	/* Profile the VM-exit. */
10463	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10464	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10465	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10466	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10467	HMVMX_START_EXIT_DISPATCH_PROF();
10468
10469	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10470
10471	/*
10472	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10473	*/
10474	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10475	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10476	if (rcStrict != VINF_SUCCESS)
10477	break;
10478	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10479	{
10480	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10481	rcStrict = VINF_EM_RAW_INTERRUPT;
10482	break;
10483	}
10484
10485	/*
10486	* Stepping: Did the RIP change, if so, consider it a single step.
10487	* Otherwise, make sure one of the TFs gets set.
10488	*/
10489	if (fStepping)
10490	{
10491	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10492	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10493	AssertRCReturn(rc2, rc2);
10494	if ( pCtx->rip != DbgState.uRipStart
10495	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10496	{
10497	rcStrict = VINF_EM_DBG_STEPPED;
10498	break;
10499	}
10500	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10501	}
10502
10503	/*
10504	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10505	*/
10506	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10507	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10508	}
10509
10510	/*
10511	* Clear the X86_EFL_TF if necessary.
10512	*/
10513	if (pVCpu->hm.s.fClearTrapFlag)
10514	{
10515	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10516	AssertRCReturn(rc2, rc2);
10517	pVCpu->hm.s.fClearTrapFlag = false;
10518	pCtx->eflags.Bits.u1TF = 0;
10519	}
10520	/** @todo there seems to be issues with the resume flag when the monitor trap
10521	* flag is pending without being used. Seen early in bios init when
10522	* accessing APIC page in protected mode. */
10523
10524	/*
10525	* Restore VM-exit control settings as we may not reenter this function the
10526	* next time around.
10527	*/
10528	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10529
10530	/* Restore HMCPU indicators. */
10531	pVCpu->hm.s.fUsingDebugLoop = false;
10532	pVCpu->hm.s.fDebugWantRdTscExit = false;
10533	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10534
10535	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10536	return rcStrict;
10537	}
10538
10539
10540	/** @} */
10541
10542
10543	/**
10544	* Checks if any expensive dtrace probes are enabled and we should go to the
10545	* debug loop.
10546	*
10547	* @returns true if we should use debug loop, false if not.
10548	*/
10549	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10550	{
10551	/* It's probably faster to OR the raw 32-bit counter variables together.
10552	Since the variables are in an array and the probes are next to one
10553	another (more or less), we have good locality. So, better read
10554	eight-nine cache lines ever time and only have one conditional, than
10555	128+ conditionals, right? */
10556	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10557	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10558	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10559	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10560	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10561	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10562	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10563	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10564	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10565	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10566	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10567	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10568	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10569	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10570	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10571	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10572	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10573	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10574	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10575	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10576	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10577	) != 0
10578	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10579	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10580	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10581	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10582	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10583	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10584	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10585	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10586	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10587	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10588	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10589	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10590	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10591	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10592	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10593	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10594	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10595	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10596	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10597	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10598	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10599	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10600	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10601	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10602	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10603	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10604	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10605	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10606	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10607	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10608	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10609	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10610	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10611	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10612	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10613	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10614	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10615	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10616	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10617	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10618	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10619	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10620	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10621	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10622	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10623	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10624	) != 0
10625	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10626	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10627	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10628	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10629	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10630	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10631	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10632	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10633	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10634	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10635	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10636	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10637	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10638	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10639	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10640	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10641	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10642	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10643	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10644	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10645	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10646	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10647	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10648	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10649	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10650	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10651	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10652	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10653	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10654	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10655	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10656	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10657	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10658	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10659	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10660	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10661	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10662	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10663	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10664	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10665	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10666	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10667	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10668	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10669	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10670	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10671	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10672	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10673	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10674	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10675	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10676	) != 0;
10677	}
10678
10679
10680	/**
10681	* Runs the guest code using VT-x.
10682	*
10683	* @returns Strict VBox status code (i.e. informational status codes too).
10684	* @param pVM The cross context VM structure.
10685	* @param pVCpu The cross context virtual CPU structure.
10686	* @param pCtx Pointer to the guest-CPU context.
10687	*/
10688	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10689	{
10690	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10691	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10692	HMVMX_ASSERT_PREEMPT_SAFE();
10693
10694	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10695
10696	VBOXSTRICTRC rcStrict;
10697	if ( !pVCpu->hm.s.fUseDebugLoop
10698	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10699	&& !DBGFIsStepping(pVCpu)
10700	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10701	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10702	else
10703	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10704
10705	if (rcStrict == VERR_EM_INTERPRETER)
10706	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10707	else if (rcStrict == VINF_EM_RESET)
10708	rcStrict = VINF_EM_TRIPLE_FAULT;
10709
10710	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10711	if (RT_FAILURE(rc2))
10712	{
10713	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10714	rcStrict = rc2;
10715	}
10716	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10717	return rcStrict;
10718	}
10719
10720
10721	#ifndef HMVMX_USE_FUNCTION_TABLE
10722	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10723	{
10724	# ifdef DEBUG_ramshankar
10725	# define RETURN_EXIT_CALL(a_CallExpr) \
10726	do { \
10727	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10728	VBOXSTRICTRC rcStrict = a_CallExpr; \
10729	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10730	return rcStrict; \
10731	} while (0)
10732	# else
10733	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10734	# endif
10735	switch (rcReason)
10736	{
10737	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10738	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10739	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10740	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10741	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10742	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10743	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10744	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10745	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10746	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10747	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10748	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10749	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10750	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10751	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10752	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10753	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10754	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10755	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10756	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10757	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10758	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10759	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10760	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10761	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10762	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10763	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10764	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10765	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10766	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10767	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10768	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10769	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10770	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10771
10772	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10773	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10774	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10775	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10776	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10777	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10778	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10779	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10780	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10781
10782	case VMX_EXIT_VMCLEAR:
10783	case VMX_EXIT_VMLAUNCH:
10784	case VMX_EXIT_VMPTRLD:
10785	case VMX_EXIT_VMPTRST:
10786	case VMX_EXIT_VMREAD:
10787	case VMX_EXIT_VMRESUME:
10788	case VMX_EXIT_VMWRITE:
10789	case VMX_EXIT_VMXOFF:
10790	case VMX_EXIT_VMXON:
10791	case VMX_EXIT_INVEPT:
10792	case VMX_EXIT_INVVPID:
10793	case VMX_EXIT_VMFUNC:
10794	case VMX_EXIT_XSAVES:
10795	case VMX_EXIT_XRSTORS:
10796	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10797	case VMX_EXIT_ENCLS:
10798	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10799	case VMX_EXIT_PML_FULL:
10800	default:
10801	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10802	}
10803	#undef RETURN_EXIT_CALL
10804	}
10805	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10806
10807
10808	#ifdef VBOX_STRICT
10809	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10810	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10811	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10812
10813	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10814	do { \
10815	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10816	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10817	} while (0)
10818
10819	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10820	do { \
10821	AssertPtr(pVCpu); \
10822	AssertPtr(pMixedCtx); \
10823	AssertPtr(pVmxTransient); \
10824	Assert(pVmxTransient->fVMEntryFailed == false); \
10825	Assert(ASMIntAreEnabled()); \
10826	HMVMX_ASSERT_PREEMPT_SAFE(); \
10827	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10828	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10829	HMVMX_ASSERT_PREEMPT_SAFE(); \
10830	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10831	HMVMX_ASSERT_PREEMPT_CPUID(); \
10832	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10833	} while (0)
10834
10835	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10836	do { \
10837	Log4Func(("\n")); \
10838	} while (0)
10839	#else /* nonstrict builds: */
10840	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10841	do { \
10842	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10843	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10844	} while (0)
10845	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10846	#endif
10847
10848
10849	/**
10850	* Advances the guest RIP by the specified number of bytes.
10851	*
10852	* @param pVCpu The cross context virtual CPU structure.
10853	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10854	* out-of-sync. Make sure to update the required fields
10855	* before using them.
10856	* @param cbInstr Number of bytes to advance the RIP by.
10857	*
10858	* @remarks No-long-jump zone!!!
10859	*/
10860	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10861	{
10862	/* Advance the RIP. */
10863	pMixedCtx->rip += cbInstr;
10864	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10865
10866	/* Update interrupt inhibition. */
10867	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10868	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10869	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10870	}
10871
10872
10873	/**
10874	* Advances the guest RIP after reading it from the VMCS.
10875	*
10876	* @returns VBox status code, no informational status codes.
10877	* @param pVCpu The cross context virtual CPU structure.
10878	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10879	* out-of-sync. Make sure to update the required fields
10880	* before using them.
10881	* @param pVmxTransient Pointer to the VMX transient structure.
10882	*
10883	* @remarks No-long-jump zone!!!
10884	*/
10885	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10886	{
10887	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10888	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10889	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10890	AssertRCReturn(rc, rc);
10891
10892	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10893
10894	/*
10895	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10896	* pending debug exception field as it takes care of priority of events.
10897	*
10898	* See Intel spec. 32.2.1 "Debug Exceptions".
10899	*/
10900	if ( !pVCpu->hm.s.fSingleInstruction
10901	&& pMixedCtx->eflags.Bits.u1TF)
10902	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10903
10904	return VINF_SUCCESS;
10905	}
10906
10907
10908	/**
10909	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10910	* and update error record fields accordingly.
10911	*
10912	* @return VMX_IGS_* return codes.
10913	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10914	* wrong with the guest state.
10915	*
10916	* @param pVM The cross context VM structure.
10917	* @param pVCpu The cross context virtual CPU structure.
10918	* @param pCtx Pointer to the guest-CPU state.
10919	*
10920	* @remarks This function assumes our cache of the VMCS controls
10921	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10922	*/
10923	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10924	{
10925	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10926	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10927	uError = (err); \
10928	break; \
10929	} else do { } while (0)
10930
10931	int rc;
10932	uint32_t uError = VMX_IGS_ERROR;
10933	uint32_t u32Val;
10934	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10935
10936	do
10937	{
10938	/*
10939	* CR0.
10940	*/
10941	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10942	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10943	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10944	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10945	if (fUnrestrictedGuest)
10946	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10947
10948	uint32_t u32GuestCR0;
10949	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10950	AssertRCBreak(rc);
10951	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10952	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10953	if ( !fUnrestrictedGuest
10954	&& (u32GuestCR0 & X86_CR0_PG)
10955	&& !(u32GuestCR0 & X86_CR0_PE))
10956	{
10957	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10958	}
10959
10960	/*
10961	* CR4.
10962	*/
10963	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10964	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10965
10966	uint32_t u32GuestCR4;
10967	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10968	AssertRCBreak(rc);
10969	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10970	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10971
10972	/*
10973	* IA32_DEBUGCTL MSR.
10974	*/
10975	uint64_t u64Val;
10976	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10977	AssertRCBreak(rc);
10978	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10979	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10980	{
10981	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10982	}
10983	uint64_t u64DebugCtlMsr = u64Val;
10984
10985	#ifdef VBOX_STRICT
10986	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10987	AssertRCBreak(rc);
10988	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10989	#endif
10990	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10991
10992	/*
10993	* RIP and RFLAGS.
10994	*/
10995	uint32_t u32Eflags;
10996	#if HC_ARCH_BITS == 64
10997	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10998	AssertRCBreak(rc);
10999	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11000	if ( !fLongModeGuest
11001	\|\| !pCtx->cs.Attr.n.u1Long)
11002	{
11003	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11004	}
11005	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11006	* must be identical if the "IA-32e mode guest" VM-entry
11007	* control is 1 and CS.L is 1. No check applies if the
11008	* CPU supports 64 linear-address bits. */
11009
11010	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11011	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11012	AssertRCBreak(rc);
11013	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11014	VMX_IGS_RFLAGS_RESERVED);
11015	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11016	u32Eflags = u64Val;
11017	#else
11018	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11019	AssertRCBreak(rc);
11020	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11021	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11022	#endif
11023
11024	if ( fLongModeGuest
11025	\|\| ( fUnrestrictedGuest
11026	&& !(u32GuestCR0 & X86_CR0_PE)))
11027	{
11028	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11029	}
11030
11031	uint32_t u32EntryInfo;
11032	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11033	AssertRCBreak(rc);
11034	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11035	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11036	{
11037	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11038	}
11039
11040	/*
11041	* 64-bit checks.
11042	*/
11043	#if HC_ARCH_BITS == 64
11044	if (fLongModeGuest)
11045	{
11046	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11047	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11048	}
11049
11050	if ( !fLongModeGuest
11051	&& (u32GuestCR4 & X86_CR4_PCIDE))
11052	{
11053	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11054	}
11055
11056	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11057	* 51:32 beyond the processor's physical-address width are 0. */
11058
11059	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11060	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11061	{
11062	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11063	}
11064
11065	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11066	AssertRCBreak(rc);
11067	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11068
11069	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11070	AssertRCBreak(rc);
11071	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11072	#endif
11073
11074	/*
11075	* PERF_GLOBAL MSR.
11076	*/
11077	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11078	{
11079	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11080	AssertRCBreak(rc);
11081	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11082	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11083	}
11084
11085	/*
11086	* PAT MSR.
11087	*/
11088	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11089	{
11090	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11091	AssertRCBreak(rc);
11092	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11093	for (unsigned i = 0; i < 8; i++)
11094	{
11095	uint8_t u8Val = (u64Val & 0xff);
11096	if ( u8Val != 0 /* UC */
11097	&& u8Val != 1 /* WC */
11098	&& u8Val != 4 /* WT */
11099	&& u8Val != 5 /* WP */
11100	&& u8Val != 6 /* WB */
11101	&& u8Val != 7 /* UC- */)
11102	{
11103	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11104	}
11105	u64Val >>= 8;
11106	}
11107	}
11108
11109	/*
11110	* EFER MSR.
11111	*/
11112	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11113	{
11114	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11115	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11116	AssertRCBreak(rc);
11117	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11118	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11119	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11120	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11121	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11122	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11123	\|\| !(u32GuestCR0 & X86_CR0_PG)
11124	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11125	VMX_IGS_EFER_LMA_LME_MISMATCH);
11126	}
11127
11128	/*
11129	* Segment registers.
11130	*/
11131	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11132	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11133	if (!(u32Eflags & X86_EFL_VM))
11134	{
11135	/* CS */
11136	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11137	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11138	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11139	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11140	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11141	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11142	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11143	/* CS cannot be loaded with NULL in protected mode. */
11144	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11145	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11146	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11147	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11148	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11149	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11150	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11151	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11152	else
11153	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11154
11155	/* SS */
11156	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11157	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11158	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11159	if ( !(pCtx->cr0 & X86_CR0_PE)
11160	\|\| pCtx->cs.Attr.n.u4Type == 3)
11161	{
11162	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11163	}
11164	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11165	{
11166	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11167	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11168	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11169	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11170	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11171	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11172	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11173	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11174	}
11175
11176	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11177	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11178	{
11179	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11180	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11181	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11182	\|\| pCtx->ds.Attr.n.u4Type > 11
11183	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11184	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11185	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11186	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11187	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11188	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11189	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11190	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11191	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11192	}
11193	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11194	{
11195	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11196	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11197	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11198	\|\| pCtx->es.Attr.n.u4Type > 11
11199	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11200	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11201	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11202	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11203	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11204	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11205	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11206	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11207	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11208	}
11209	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11210	{
11211	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11212	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11213	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11214	\|\| pCtx->fs.Attr.n.u4Type > 11
11215	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11216	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11217	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11218	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11219	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11220	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11221	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11222	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11223	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11224	}
11225	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11226	{
11227	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11228	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11229	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11230	\|\| pCtx->gs.Attr.n.u4Type > 11
11231	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11232	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11233	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11234	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11235	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11236	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11237	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11238	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11239	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11240	}
11241	/* 64-bit capable CPUs. */
11242	#if HC_ARCH_BITS == 64
11243	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11244	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11245	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11246	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11247	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11248	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11249	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11250	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11251	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11252	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11253	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11254	#endif
11255	}
11256	else
11257	{
11258	/* V86 mode checks. */
11259	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11260	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11261	{
11262	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11263	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11264	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11265	}
11266	else
11267	{
11268	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11269	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11270	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11271	}
11272
11273	/* CS */
11274	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11275	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11276	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11277	/* SS */
11278	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11279	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11280	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11281	/* DS */
11282	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11283	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11284	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11285	/* ES */
11286	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11287	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11288	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11289	/* FS */
11290	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11291	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11292	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11293	/* GS */
11294	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11295	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11296	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11297	/* 64-bit capable CPUs. */
11298	#if HC_ARCH_BITS == 64
11299	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11300	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11301	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11302	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11303	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11304	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11305	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11306	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11307	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11308	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11309	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11310	#endif
11311	}
11312
11313	/*
11314	* TR.
11315	*/
11316	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11317	/* 64-bit capable CPUs. */
11318	#if HC_ARCH_BITS == 64
11319	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11320	#endif
11321	if (fLongModeGuest)
11322	{
11323	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11324	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11325	}
11326	else
11327	{
11328	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11329	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11330	VMX_IGS_TR_ATTR_TYPE_INVALID);
11331	}
11332	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11333	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11334	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11335	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11336	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11337	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11338	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11339	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11340
11341	/*
11342	* GDTR and IDTR.
11343	*/
11344	#if HC_ARCH_BITS == 64
11345	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11346	AssertRCBreak(rc);
11347	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11348
11349	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11350	AssertRCBreak(rc);
11351	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11352	#endif
11353
11354	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11355	AssertRCBreak(rc);
11356	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11357
11358	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11359	AssertRCBreak(rc);
11360	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11361
11362	/*
11363	* Guest Non-Register State.
11364	*/
11365	/* Activity State. */
11366	uint32_t u32ActivityState;
11367	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11368	AssertRCBreak(rc);
11369	HMVMX_CHECK_BREAK( !u32ActivityState
11370	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11371	VMX_IGS_ACTIVITY_STATE_INVALID);
11372	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11373	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11374	uint32_t u32IntrState;
11375	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11376	AssertRCBreak(rc);
11377	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11378	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11379	{
11380	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11381	}
11382
11383	/** @todo Activity state and injecting interrupts. Left as a todo since we
11384	* currently don't use activity states but ACTIVE. */
11385
11386	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11387	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11388
11389	/* Guest interruptibility-state. */
11390	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11391	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11392	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11393	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11394	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11395	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11396	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11397	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11398	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11399	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11400	{
11401	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11402	{
11403	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11404	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11405	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11406	}
11407	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11408	{
11409	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11410	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11411	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11412	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11413	}
11414	}
11415	/** @todo Assumes the processor is not in SMM. */
11416	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11417	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11418	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11419	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11420	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11421	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11422	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11423	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11424	{
11425	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11426	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11427	}
11428
11429	/* Pending debug exceptions. */
11430	#if HC_ARCH_BITS == 64
11431	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11432	AssertRCBreak(rc);
11433	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11434	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11435	u32Val = u64Val; /* For pending debug exceptions checks below. */
11436	#else
11437	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11438	AssertRCBreak(rc);
11439	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11440	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11441	#endif
11442
11443	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11444	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11445	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11446	{
11447	if ( (u32Eflags & X86_EFL_TF)
11448	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11449	{
11450	/* Bit 14 is PendingDebug.BS. */
11451	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11452	}
11453	if ( !(u32Eflags & X86_EFL_TF)
11454	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11455	{
11456	/* Bit 14 is PendingDebug.BS. */
11457	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11458	}
11459	}
11460
11461	/* VMCS link pointer. */
11462	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11463	AssertRCBreak(rc);
11464	if (u64Val != UINT64_C(0xffffffffffffffff))
11465	{
11466	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11467	/** @todo Bits beyond the processor's physical-address width MBZ. */
11468	/** @todo 32-bit located in memory referenced by value of this field (as a
11469	* physical address) must contain the processor's VMCS revision ID. */
11470	/** @todo SMM checks. */
11471	}
11472
11473	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11474	* not using Nested Paging? */
11475	if ( pVM->hm.s.fNestedPaging
11476	&& !fLongModeGuest
11477	&& CPUMIsGuestInPAEModeEx(pCtx))
11478	{
11479	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11480	AssertRCBreak(rc);
11481	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11482
11483	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11484	AssertRCBreak(rc);
11485	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11486
11487	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11488	AssertRCBreak(rc);
11489	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11490
11491	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11492	AssertRCBreak(rc);
11493	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11494	}
11495
11496	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11497	if (uError == VMX_IGS_ERROR)
11498	uError = VMX_IGS_REASON_NOT_FOUND;
11499	} while (0);
11500
11501	pVCpu->hm.s.u32HMError = uError;
11502	return uError;
11503
11504	#undef HMVMX_ERROR_BREAK
11505	#undef HMVMX_CHECK_BREAK
11506	}
11507
11508	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11509	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11510	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11511
11512	/** @name VM-exit handlers.
11513	* @{
11514	*/
11515
11516	/**
11517	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11518	*/
11519	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11520	{
11521	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11522	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11523	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11524	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11525	return VINF_SUCCESS;
11526	return VINF_EM_RAW_INTERRUPT;
11527	}
11528
11529
11530	/**
11531	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11532	*/
11533	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11534	{
11535	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11536	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11537
11538	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11539	AssertRCReturn(rc, rc);
11540
11541	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11542	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11543	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11544	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11545
11546	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11547	{
11548	/*
11549	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11550	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11551	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11552	*
11553	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11554	*/
11555	VMXDispatchHostNmi();
11556	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11557	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11558	return VINF_SUCCESS;
11559	}
11560
11561	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11562	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11563	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11564	{ /* likely */ }
11565	else
11566	{
11567	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11568	rcStrictRc1 = VINF_SUCCESS;
11569	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11570	return rcStrictRc1;
11571	}
11572
11573	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11574	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11575	switch (uIntType)
11576	{
11577	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11578	Assert(uVector == X86_XCPT_DB);
11579	/* fall thru */
11580	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11581	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11582	/* fall thru */
11583	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11584	{
11585	/*
11586	* If there's any exception caused as a result of event injection, the resulting
11587	* secondary/final execption will be pending, we shall continue guest execution
11588	* after injecting the event. The page-fault case is complicated and we manually
11589	* handle any currently pending event in hmR0VmxExitXcptPF.
11590	*/
11591	if (!pVCpu->hm.s.Event.fPending)
11592	{ /* likely */ }
11593	else if (uVector != X86_XCPT_PF)
11594	{
11595	rc = VINF_SUCCESS;
11596	break;
11597	}
11598
11599	switch (uVector)
11600	{
11601	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11602	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11603	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11604	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11605	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11606	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11607	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11608
11609	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11610	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11611	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11612	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11613	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11614	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11615	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11616	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11617	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11618	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11619	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11620	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11621	default:
11622	{
11623	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11624	AssertRCReturn(rc, rc);
11625
11626	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11627	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11628	{
11629	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11630	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11631	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11632
11633	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11634	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11635	AssertRCReturn(rc, rc);
11636	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11637	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11638	0 /* GCPtrFaultAddress */);
11639	AssertRCReturn(rc, rc);
11640	}
11641	else
11642	{
11643	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11644	pVCpu->hm.s.u32HMError = uVector;
11645	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11646	}
11647	break;
11648	}
11649	}
11650	break;
11651	}
11652
11653	default:
11654	{
11655	pVCpu->hm.s.u32HMError = uExitIntInfo;
11656	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11657	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11658	break;
11659	}
11660	}
11661	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11662	return rc;
11663	}
11664
11665
11666	/**
11667	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11668	*/
11669	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11670	{
11671	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11672
11673	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11674	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11675
11676	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11677	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11678	return VINF_SUCCESS;
11679	}
11680
11681
11682	/**
11683	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11684	*/
11685	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11686	{
11687	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11688	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11689	{
11690	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11691	HMVMX_RETURN_UNEXPECTED_EXIT();
11692	}
11693
11694	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11695
11696	/*
11697	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11698	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11699	*/
11700	uint32_t uIntrState = 0;
11701	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11702	AssertRCReturn(rc, rc);
11703
11704	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11705	if ( fBlockSti
11706	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11707	{
11708	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11709	}
11710
11711	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11712	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11713
11714	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11715	return VINF_SUCCESS;
11716	}
11717
11718
11719	/**
11720	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11721	*/
11722	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11723	{
11724	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11725	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11726	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11727	}
11728
11729
11730	/**
11731	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11732	*/
11733	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11734	{
11735	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11736	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11737	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11738	}
11739
11740
11741	/**
11742	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11743	*/
11744	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11745	{
11746	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11747	PVM pVM = pVCpu->CTX_SUFF(pVM);
11748	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11749	if (RT_LIKELY(rc == VINF_SUCCESS))
11750	{
11751	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11752	Assert(pVmxTransient->cbInstr == 2);
11753	}
11754	else
11755	{
11756	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11757	rc = VERR_EM_INTERPRETER;
11758	}
11759	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11760	return rc;
11761	}
11762
11763
11764	/**
11765	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11766	*/
11767	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11768	{
11769	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11770	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11771	AssertRCReturn(rc, rc);
11772
11773	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11774	return VINF_EM_RAW_EMULATE_INSTR;
11775
11776	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11777	HMVMX_RETURN_UNEXPECTED_EXIT();
11778	}
11779
11780
11781	/**
11782	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11783	*/
11784	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11785	{
11786	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11787	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11788	AssertRCReturn(rc, rc);
11789
11790	PVM pVM = pVCpu->CTX_SUFF(pVM);
11791	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11792	if (RT_LIKELY(rc == VINF_SUCCESS))
11793	{
11794	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11795	Assert(pVmxTransient->cbInstr == 2);
11796	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11797	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11798	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11799	}
11800	else
11801	rc = VERR_EM_INTERPRETER;
11802	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11803	return rc;
11804	}
11805
11806
11807	/**
11808	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11809	*/
11810	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11811	{
11812	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11813	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11814	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11815	AssertRCReturn(rc, rc);
11816
11817	PVM pVM = pVCpu->CTX_SUFF(pVM);
11818	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11819	if (RT_SUCCESS(rc))
11820	{
11821	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11822	Assert(pVmxTransient->cbInstr == 3);
11823	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11824	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11825	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11826	}
11827	else
11828	{
11829	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11830	rc = VERR_EM_INTERPRETER;
11831	}
11832	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11833	return rc;
11834	}
11835
11836
11837	/**
11838	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11839	*/
11840	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11841	{
11842	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11843	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11844	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11845	AssertRCReturn(rc, rc);
11846
11847	PVM pVM = pVCpu->CTX_SUFF(pVM);
11848	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11849	if (RT_LIKELY(rc == VINF_SUCCESS))
11850	{
11851	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11852	Assert(pVmxTransient->cbInstr == 2);
11853	}
11854	else
11855	{
11856	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11857	rc = VERR_EM_INTERPRETER;
11858	}
11859	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11860	return rc;
11861	}
11862
11863
11864	/**
11865	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11866	*/
11867	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11868	{
11869	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11870	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11871
11872	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11873	if (pVCpu->hm.s.fHypercallsEnabled)
11874	{
11875	#if 0
11876	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11877	#else
11878	/* Aggressive state sync. for now. */
11879	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11880	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11881	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11882	AssertRCReturn(rc, rc);
11883	#endif
11884
11885	/* Perform the hypercall. */
11886	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11887	if (rcStrict == VINF_SUCCESS)
11888	{
11889	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11890	AssertRCReturn(rc, rc);
11891	}
11892	else
11893	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11894	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11895	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11896
11897	/* If the hypercall changes anything other than guest's general-purpose registers,
11898	we would need to reload the guest changed bits here before VM-entry. */
11899	}
11900	else
11901	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11902
11903	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11904	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11905	{
11906	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11907	rcStrict = VINF_SUCCESS;
11908	}
11909
11910	return rcStrict;
11911	}
11912
11913
11914	/**
11915	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11916	*/
11917	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11918	{
11919	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11920	PVM pVM = pVCpu->CTX_SUFF(pVM);
11921	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11922
11923	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11924	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11925	AssertRCReturn(rc, rc);
11926
11927	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11928	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11929	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11930	else
11931	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11932	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11934	return rcStrict;
11935	}
11936
11937
11938	/**
11939	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11940	*/
11941	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11942	{
11943	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11944	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11945	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11946	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11947	AssertRCReturn(rc, rc);
11948
11949	PVM pVM = pVCpu->CTX_SUFF(pVM);
11950	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11951	if (RT_LIKELY(rc == VINF_SUCCESS))
11952	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11953	else
11954	{
11955	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11956	rc = VERR_EM_INTERPRETER;
11957	}
11958	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11959	return rc;
11960	}
11961
11962
11963	/**
11964	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11965	*/
11966	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11967	{
11968	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11969	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11970	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11971	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11972	AssertRCReturn(rc, rc);
11973
11974	PVM pVM = pVCpu->CTX_SUFF(pVM);
11975	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11976	rc = VBOXSTRICTRC_VAL(rc2);
11977	if (RT_LIKELY( rc == VINF_SUCCESS
11978	\|\| rc == VINF_EM_HALT))
11979	{
11980	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11981	AssertRCReturn(rc3, rc3);
11982
11983	if ( rc == VINF_EM_HALT
11984	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11985	{
11986	rc = VINF_SUCCESS;
11987	}
11988	}
11989	else
11990	{
11991	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11992	rc = VERR_EM_INTERPRETER;
11993	}
11994	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11995	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11996	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11997	return rc;
11998	}
11999
12000
12001	/**
12002	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12003	*/
12004	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12005	{
12006	/*
12007	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12008	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12009	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
12010	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12011	*/
12012	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12013	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12014	HMVMX_RETURN_UNEXPECTED_EXIT();
12015	}
12016
12017
12018	/**
12019	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12020	*/
12021	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12022	{
12023	/*
12024	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12025	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12026	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12027	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12028	*/
12029	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12030	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12031	HMVMX_RETURN_UNEXPECTED_EXIT();
12032	}
12033
12034
12035	/**
12036	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12037	*/
12038	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12039	{
12040	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12041	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12042	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12043	HMVMX_RETURN_UNEXPECTED_EXIT();
12044	}
12045
12046
12047	/**
12048	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12049	*/
12050	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12051	{
12052	/*
12053	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12054	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12055	* See Intel spec. 25.3 "Other Causes of VM-exits".
12056	*/
12057	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12058	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12059	HMVMX_RETURN_UNEXPECTED_EXIT();
12060	}
12061
12062
12063	/**
12064	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12065	* VM-exit.
12066	*/
12067	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12068	{
12069	/*
12070	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12071	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12072	*
12073	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12074	* See Intel spec. "23.8 Restrictions on VMX operation".
12075	*/
12076	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12077	return VINF_SUCCESS;
12078	}
12079
12080
12081	/**
12082	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12083	* VM-exit.
12084	*/
12085	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12086	{
12087	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12088	return VINF_EM_RESET;
12089	}
12090
12091
12092	/**
12093	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12094	*/
12095	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12096	{
12097	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12098	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12099
12100	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12101	AssertRCReturn(rc, rc);
12102
12103	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12104	rc = VINF_SUCCESS;
12105	else
12106	rc = VINF_EM_HALT;
12107
12108	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12109	if (rc != VINF_SUCCESS)
12110	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12111	return rc;
12112	}
12113
12114
12115	/**
12116	* VM-exit handler for instructions that result in a \#UD exception delivered to
12117	* the guest.
12118	*/
12119	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12120	{
12121	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12122	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12123	return VINF_SUCCESS;
12124	}
12125
12126
12127	/**
12128	* VM-exit handler for expiry of the VMX preemption timer.
12129	*/
12130	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12131	{
12132	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12133
12134	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12135	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12136
12137	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12138	PVM pVM = pVCpu->CTX_SUFF(pVM);
12139	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12140	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12141	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12142	}
12143
12144
12145	/**
12146	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12147	*/
12148	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12149	{
12150	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12151
12152	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12153	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12154	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12155	AssertRCReturn(rc, rc);
12156
12157	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12158	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12159
12160	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12161
12162	return rcStrict;
12163	}
12164
12165
12166	/**
12167	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12168	*/
12169	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12170	{
12171	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12172
12173	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
12174	/** @todo implement EMInterpretInvpcid() */
12175	return VERR_EM_INTERPRETER;
12176	}
12177
12178
12179	/**
12180	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12181	* Error VM-exit.
12182	*/
12183	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12184	{
12185	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12186	AssertRCReturn(rc, rc);
12187
12188	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12189	AssertRCReturn(rc, rc);
12190
12191	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12192	NOREF(uInvalidReason);
12193
12194	#ifdef VBOX_STRICT
12195	uint32_t uIntrState;
12196	RTHCUINTREG uHCReg;
12197	uint64_t u64Val;
12198	uint32_t u32Val;
12199
12200	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12201	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12202	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12203	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12204	AssertRCReturn(rc, rc);
12205
12206	Log4(("uInvalidReason %u\n", uInvalidReason));
12207	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12208	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12209	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12210	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12211
12212	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12213	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12214	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12215	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12216	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12217	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12218	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12219	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12220	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12221	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12222	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12223	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12224	#else
12225	NOREF(pVmxTransient);
12226	#endif
12227
12228	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12229	return VERR_VMX_INVALID_GUEST_STATE;
12230	}
12231
12232
12233	/**
12234	* VM-exit handler for VM-entry failure due to an MSR-load
12235	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12236	*/
12237	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12238	{
12239	NOREF(pVmxTransient);
12240	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12241	HMVMX_RETURN_UNEXPECTED_EXIT();
12242	}
12243
12244
12245	/**
12246	* VM-exit handler for VM-entry failure due to a machine-check event
12247	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12248	*/
12249	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12250	{
12251	NOREF(pVmxTransient);
12252	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12253	HMVMX_RETURN_UNEXPECTED_EXIT();
12254	}
12255
12256
12257	/**
12258	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12259	* theory.
12260	*/
12261	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12262	{
12263	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12264	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12265	return VERR_VMX_UNDEFINED_EXIT_CODE;
12266	}
12267
12268
12269	/**
12270	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12271	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12272	* Conditional VM-exit.
12273	*/
12274	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12275	{
12276	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12277
12278	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12280	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12281	return VERR_EM_INTERPRETER;
12282	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12283	HMVMX_RETURN_UNEXPECTED_EXIT();
12284	}
12285
12286
12287	/**
12288	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12289	*/
12290	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12291	{
12292	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12293
12294	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12295	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12296	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12297	return VERR_EM_INTERPRETER;
12298	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12299	HMVMX_RETURN_UNEXPECTED_EXIT();
12300	}
12301
12302
12303	/**
12304	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12305	*/
12306	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12307	{
12308	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12309
12310	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12311	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12312	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12313	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12314	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12315	{
12316	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12317	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12318	}
12319	AssertRCReturn(rc, rc);
12320	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12321
12322	#ifdef VBOX_STRICT
12323	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12324	{
12325	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12326	&& pMixedCtx->ecx != MSR_K6_EFER)
12327	{
12328	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12329	pMixedCtx->ecx));
12330	HMVMX_RETURN_UNEXPECTED_EXIT();
12331	}
12332	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12333	{
12334	VMXMSREXITREAD enmRead;
12335	VMXMSREXITWRITE enmWrite;
12336	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12337	AssertRCReturn(rc2, rc2);
12338	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12339	{
12340	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12341	HMVMX_RETURN_UNEXPECTED_EXIT();
12342	}
12343	}
12344	}
12345	#endif
12346
12347	PVM pVM = pVCpu->CTX_SUFF(pVM);
12348	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12349	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12350	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12351	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12352	if (RT_SUCCESS(rc))
12353	{
12354	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12355	Assert(pVmxTransient->cbInstr == 2);
12356	}
12357	return rc;
12358	}
12359
12360
12361	/**
12362	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12363	*/
12364	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12365	{
12366	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12367	PVM pVM = pVCpu->CTX_SUFF(pVM);
12368	int rc = VINF_SUCCESS;
12369
12370	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12371	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12372	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12373	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12374	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12375	{
12376	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12377	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12378	}
12379	AssertRCReturn(rc, rc);
12380	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12381
12382	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12383	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12385
12386	if (RT_SUCCESS(rc))
12387	{
12388	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12389
12390	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12391	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12392	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12393	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12394	{
12395	/*
12396	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12397	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12398	* EMInterpretWrmsr() changes it.
12399	*/
12400	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12401	}
12402	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12403	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12404	else if (pMixedCtx->ecx == MSR_K6_EFER)
12405	{
12406	/*
12407	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12408	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12409	* the other bits as well, SCE and NXE. See @bugref{7368}.
12410	*/
12411	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12412	}
12413
12414	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12415	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12416	{
12417	switch (pMixedCtx->ecx)
12418	{
12419	/*
12420	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12421	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12422	*/
12423	case MSR_IA32_SYSENTER_CS:
12424	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12425	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12426	break;
12427	case MSR_IA32_SYSENTER_EIP:
12428	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12429	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12430	break;
12431	case MSR_IA32_SYSENTER_ESP:
12432	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12433	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12434	break;
12435	case MSR_K8_FS_BASE: /* fall thru */
12436	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12437	case MSR_K6_EFER: /* already handled above */ break;
12438	default:
12439	{
12440	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12441	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12442	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12443	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12444	break;
12445	}
12446	}
12447	}
12448	#ifdef VBOX_STRICT
12449	else
12450	{
12451	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12452	switch (pMixedCtx->ecx)
12453	{
12454	case MSR_IA32_SYSENTER_CS:
12455	case MSR_IA32_SYSENTER_EIP:
12456	case MSR_IA32_SYSENTER_ESP:
12457	case MSR_K8_FS_BASE:
12458	case MSR_K8_GS_BASE:
12459	{
12460	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12461	HMVMX_RETURN_UNEXPECTED_EXIT();
12462	}
12463
12464	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12465	default:
12466	{
12467	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12468	{
12469	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12470	if (pMixedCtx->ecx != MSR_K6_EFER)
12471	{
12472	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12473	pMixedCtx->ecx));
12474	HMVMX_RETURN_UNEXPECTED_EXIT();
12475	}
12476	}
12477
12478	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12479	{
12480	VMXMSREXITREAD enmRead;
12481	VMXMSREXITWRITE enmWrite;
12482	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12483	AssertRCReturn(rc2, rc2);
12484	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12485	{
12486	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12487	HMVMX_RETURN_UNEXPECTED_EXIT();
12488	}
12489	}
12490	break;
12491	}
12492	}
12493	}
12494	#endif /* VBOX_STRICT */
12495	}
12496	return rc;
12497	}
12498
12499
12500	/**
12501	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12502	*/
12503	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12504	{
12505	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12506
12507	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12508	return VINF_EM_RAW_INTERRUPT;
12509	}
12510
12511
12512	/**
12513	* VM-exit handler for when the TPR value is lowered below the specified
12514	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12515	*/
12516	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12517	{
12518	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12519	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12520
12521	/*
12522	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12523	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12524	*/
12525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12526	return VINF_SUCCESS;
12527	}
12528
12529
12530	/**
12531	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12532	* VM-exit.
12533	*
12534	* @retval VINF_SUCCESS when guest execution can continue.
12535	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12536	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12537	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12538	* interpreter.
12539	*/
12540	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12541	{
12542	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12543	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12544	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12545	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12546	AssertRCReturn(rc, rc);
12547
12548	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12549	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12550	PVM pVM = pVCpu->CTX_SUFF(pVM);
12551	VBOXSTRICTRC rcStrict;
12552	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12553	switch (uAccessType)
12554	{
12555	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12556	{
12557	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12558	AssertRCReturn(rc, rc);
12559
12560	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12561	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12562	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12563	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12564	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12565	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12566	{
12567	case 0: /* CR0 */
12568	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12569	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12570	break;
12571	case 2: /* CR2 */
12572	/* Nothing to do here, CR2 it's not part of the VMCS. */
12573	break;
12574	case 3: /* CR3 */
12575	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12576	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12577	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12578	break;
12579	case 4: /* CR4 */
12580	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12581	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12582	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12583	break;
12584	case 8: /* CR8 */
12585	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12586	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12587	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12588	break;
12589	default:
12590	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12591	break;
12592	}
12593
12594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12595	break;
12596	}
12597
12598	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12599	{
12600	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12601	AssertRCReturn(rc, rc);
12602
12603	Assert( !pVM->hm.s.fNestedPaging
12604	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12605	\|\| pVCpu->hm.s.fUsingDebugLoop
12606	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12607
12608	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12609	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12610	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12611
12612	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12613	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12614	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12615	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12617	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12618	VBOXSTRICTRC_VAL(rcStrict)));
12619	break;
12620	}
12621
12622	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12623	{
12624	AssertRCReturn(rc, rc);
12625	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12626	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12627	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12629	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12630	break;
12631	}
12632
12633	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12634	{
12635	AssertRCReturn(rc, rc);
12636	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12637	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12638	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12639	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12640	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12641	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12642	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12643	break;
12644	}
12645
12646	default:
12647	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12648	VERR_VMX_UNEXPECTED_EXCEPTION);
12649	}
12650
12651	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12652	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12653	NOREF(pVM);
12654	return rcStrict;
12655	}
12656
12657
12658	/**
12659	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12660	* VM-exit.
12661	*/
12662	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12663	{
12664	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12665	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12666
12667	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12668	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12669	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12670	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12671	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12672	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12673	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12674	AssertRCReturn(rc, rc);
12675
12676	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12677	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12678	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12679	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12680	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12681	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12682	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12683	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12684	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12685
12686	/* I/O operation lookup arrays. */
12687	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12688	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12689
12690	VBOXSTRICTRC rcStrict;
12691	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12692	uint32_t const cbInstr = pVmxTransient->cbInstr;
12693	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12694	PVM pVM = pVCpu->CTX_SUFF(pVM);
12695	if (fIOString)
12696	{
12697	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12698	See @bugref{5752#c158}. Should work now. */
12699	/*
12700	* INS/OUTS - I/O String instruction.
12701	*
12702	* Use instruction-information if available, otherwise fall back on
12703	* interpreting the instruction.
12704	*/
12705	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12706	fIOWrite ? 'w' : 'r'));
12707	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12708	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12709	{
12710	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12711	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12712	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12713	AssertRCReturn(rc2, rc2);
12714	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12715	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12716	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12717	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12718	if (fIOWrite)
12719	{
12720	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12721	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12722	}
12723	else
12724	{
12725	/*
12726	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12727	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12728	* See Intel Instruction spec. for "INS".
12729	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12730	*/
12731	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12732	}
12733	}
12734	else
12735	{
12736	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12737	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12738	AssertRCReturn(rc2, rc2);
12739	rcStrict = IEMExecOne(pVCpu);
12740	}
12741	/** @todo IEM needs to be setting these flags somehow. */
12742	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12743	fUpdateRipAlready = true;
12744	#else
12745	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12746	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12747	if (RT_SUCCESS(rcStrict))
12748	{
12749	if (fIOWrite)
12750	{
12751	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12752	(DISCPUMODE)pDis->uAddrMode, cbValue);
12753	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12754	}
12755	else
12756	{
12757	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12758	(DISCPUMODE)pDis->uAddrMode, cbValue);
12759	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12760	}
12761	}
12762	else
12763	{
12764	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12765	pMixedCtx->rip));
12766	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12767	}
12768	#endif
12769	}
12770	else
12771	{
12772	/*
12773	* IN/OUT - I/O instruction.
12774	*/
12775	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12776	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12777	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12778	if (fIOWrite)
12779	{
12780	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12782	}
12783	else
12784	{
12785	uint32_t u32Result = 0;
12786	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12787	if (IOM_SUCCESS(rcStrict))
12788	{
12789	/* Save result of I/O IN instr. in AL/AX/EAX. */
12790	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12791	}
12792	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12793	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12795	}
12796	}
12797
12798	if (IOM_SUCCESS(rcStrict))
12799	{
12800	if (!fUpdateRipAlready)
12801	{
12802	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12803	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12804	}
12805
12806	/*
12807	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12808	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12809	*/
12810	if (fIOString)
12811	{
12812	/** @todo Single-step for INS/OUTS with REP prefix? */
12813	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12814	}
12815	else if ( !fDbgStepping
12816	&& fGstStepping)
12817	{
12818	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12819	}
12820
12821	/*
12822	* If any I/O breakpoints are armed, we need to check if one triggered
12823	* and take appropriate action.
12824	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12825	*/
12826	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12827	AssertRCReturn(rc2, rc2);
12828
12829	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12830	* execution engines about whether hyper BPs and such are pending. */
12831	uint32_t const uDr7 = pMixedCtx->dr[7];
12832	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12833	&& X86_DR7_ANY_RW_IO(uDr7)
12834	&& (pMixedCtx->cr4 & X86_CR4_DE))
12835	\|\| DBGFBpIsHwIoArmed(pVM)))
12836	{
12837	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12838
12839	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12840	VMMRZCallRing3Disable(pVCpu);
12841	HM_DISABLE_PREEMPT();
12842
12843	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12844
12845	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12846	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12847	{
12848	/* Raise #DB. */
12849	if (fIsGuestDbgActive)
12850	ASMSetDR6(pMixedCtx->dr[6]);
12851	if (pMixedCtx->dr[7] != uDr7)
12852	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12853
12854	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12855	}
12856	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12857	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12858	else if ( rcStrict2 != VINF_SUCCESS
12859	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12860	rcStrict = rcStrict2;
12861	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12862
12863	HM_RESTORE_PREEMPT();
12864	VMMRZCallRing3Enable(pVCpu);
12865	}
12866	}
12867
12868	#ifdef VBOX_STRICT
12869	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12870	Assert(!fIOWrite);
12871	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12872	Assert(fIOWrite);
12873	else
12874	{
12875	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12876	* statuses, that the VMM device and some others may return. See
12877	* IOM_SUCCESS() for guidance. */
12878	AssertMsg( RT_FAILURE(rcStrict)
12879	\|\| rcStrict == VINF_SUCCESS
12880	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12881	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12882	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12883	\|\| rcStrict == VINF_EM_RAW_TO_R3
12884	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12885	#endif
12886	}
12887	#endif
12888
12889	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12890	return rcStrict;
12891	}
12892
12893
12894	/**
12895	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12896	* VM-exit.
12897	*/
12898	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12899	{
12900	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12901
12902	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12903	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12904	AssertRCReturn(rc, rc);
12905	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12906	{
12907	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12908	AssertRCReturn(rc, rc);
12909	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12910	{
12911	uint32_t uErrCode;
12912	RTGCUINTPTR GCPtrFaultAddress;
12913	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12914	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12915	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12916	if (fErrorCodeValid)
12917	{
12918	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12919	AssertRCReturn(rc, rc);
12920	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12921	}
12922	else
12923	uErrCode = 0;
12924
12925	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12926	&& uVector == X86_XCPT_PF)
12927	GCPtrFaultAddress = pMixedCtx->cr2;
12928	else
12929	GCPtrFaultAddress = 0;
12930
12931	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12932	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12933
12934	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12935	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12936	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12937	}
12938	}
12939
12940	/* Fall back to the interpreter to emulate the task-switch. */
12941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12942	return VERR_EM_INTERPRETER;
12943	}
12944
12945
12946	/**
12947	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12948	*/
12949	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12950	{
12951	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12952	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12953	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12954	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12955	AssertRCReturn(rc, rc);
12956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12957	return VINF_EM_DBG_STEPPED;
12958	}
12959
12960
12961	/**
12962	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12963	*/
12964	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12965	{
12966	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12967
12968	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12969
12970	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12971	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12972	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12973	{
12974	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12975	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12976	{
12977	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12978	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12979	}
12980	}
12981	else
12982	{
12983	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12984	rcStrict1 = VINF_SUCCESS;
12985	return rcStrict1;
12986	}
12987
12988	#if 0
12989	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12990	* just sync the whole thing. */
12991	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12992	#else
12993	/* Aggressive state sync. for now. */
12994	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12995	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12996	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12997	#endif
12998	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12999	AssertRCReturn(rc, rc);
13000
13001	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13002	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13003	VBOXSTRICTRC rcStrict2;
13004	switch (uAccessType)
13005	{
13006	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13007	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13008	{
13009	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13010	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13011	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13012
13013	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13014	GCPhys &= PAGE_BASE_GC_MASK;
13015	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13016	PVM pVM = pVCpu->CTX_SUFF(pVM);
13017	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13018	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13019
13020	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13021	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13022	CPUMCTX2CORE(pMixedCtx), GCPhys);
13023	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13024	if ( rcStrict2 == VINF_SUCCESS
13025	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13026	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13027	{
13028	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13029	\| HM_CHANGED_GUEST_RSP
13030	\| HM_CHANGED_GUEST_RFLAGS
13031	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13032	rcStrict2 = VINF_SUCCESS;
13033	}
13034	break;
13035	}
13036
13037	default:
13038	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13039	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13040	break;
13041	}
13042
13043	if (rcStrict2 != VINF_SUCCESS)
13044	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13045	return rcStrict2;
13046	}
13047
13048
13049	/**
13050	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13051	* VM-exit.
13052	*/
13053	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13054	{
13055	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13056
13057	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13058	if (pVmxTransient->fWasGuestDebugStateActive)
13059	{
13060	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13061	HMVMX_RETURN_UNEXPECTED_EXIT();
13062	}
13063
13064	if ( !pVCpu->hm.s.fSingleInstruction
13065	&& !pVmxTransient->fWasHyperDebugStateActive)
13066	{
13067	Assert(!DBGFIsStepping(pVCpu));
13068	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13069
13070	/* Don't intercept MOV DRx any more. */
13071	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13072	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13073	AssertRCReturn(rc, rc);
13074
13075	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13076	VMMRZCallRing3Disable(pVCpu);
13077	HM_DISABLE_PREEMPT();
13078
13079	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13080	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13081	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13082
13083	HM_RESTORE_PREEMPT();
13084	VMMRZCallRing3Enable(pVCpu);
13085
13086	#ifdef VBOX_WITH_STATISTICS
13087	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13088	AssertRCReturn(rc, rc);
13089	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13091	else
13092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13093	#endif
13094	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13095	return VINF_SUCCESS;
13096	}
13097
13098	/*
13099	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13100	* Update the segment registers and DR7 from the CPU.
13101	*/
13102	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13103	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13104	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13105	AssertRCReturn(rc, rc);
13106	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13107
13108	PVM pVM = pVCpu->CTX_SUFF(pVM);
13109	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13110	{
13111	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13112	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13113	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13114	if (RT_SUCCESS(rc))
13115	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13117	}
13118	else
13119	{
13120	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13121	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13122	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13124	}
13125
13126	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13127	if (RT_SUCCESS(rc))
13128	{
13129	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13130	AssertRCReturn(rc2, rc2);
13131	return VINF_SUCCESS;
13132	}
13133	return rc;
13134	}
13135
13136
13137	/**
13138	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13139	* Conditional VM-exit.
13140	*/
13141	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13142	{
13143	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13144	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13145
13146	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13147	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13148	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13149	{
13150	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13151	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13152	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13153	{
13154	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13155	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13156	}
13157	}
13158	else
13159	{
13160	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13161	rcStrict1 = VINF_SUCCESS;
13162	return rcStrict1;
13163	}
13164
13165	RTGCPHYS GCPhys = 0;
13166	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13167
13168	#if 0
13169	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13170	#else
13171	/* Aggressive state sync. for now. */
13172	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13173	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13174	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13175	#endif
13176	AssertRCReturn(rc, rc);
13177
13178	/*
13179	* If we succeed, resume guest execution.
13180	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13181	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13182	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13183	* weird case. See @bugref{6043}.
13184	*/
13185	PVM pVM = pVCpu->CTX_SUFF(pVM);
13186	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13187	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13188	if ( rcStrict2 == VINF_SUCCESS
13189	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13190	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13191	{
13192	/* Successfully handled MMIO operation. */
13193	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13194	\| HM_CHANGED_GUEST_RSP
13195	\| HM_CHANGED_GUEST_RFLAGS
13196	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13197	return VINF_SUCCESS;
13198	}
13199	return rcStrict2;
13200	}
13201
13202
13203	/**
13204	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13205	* VM-exit.
13206	*/
13207	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13208	{
13209	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13210	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13211
13212	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13213	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13214	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13215	{
13216	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13217	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13218	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13219	}
13220	else
13221	{
13222	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13223	rcStrict1 = VINF_SUCCESS;
13224	return rcStrict1;
13225	}
13226
13227	RTGCPHYS GCPhys = 0;
13228	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13229	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13230	#if 0
13231	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13232	#else
13233	/* Aggressive state sync. for now. */
13234	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13235	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13236	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13237	#endif
13238	AssertRCReturn(rc, rc);
13239
13240	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13241	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13242
13243	RTGCUINT uErrorCode = 0;
13244	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13245	uErrorCode \|= X86_TRAP_PF_ID;
13246	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13247	uErrorCode \|= X86_TRAP_PF_RW;
13248	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13249	uErrorCode \|= X86_TRAP_PF_P;
13250
13251	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13252
13253	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13254	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13255
13256	/* Handle the pagefault trap for the nested shadow table. */
13257	PVM pVM = pVCpu->CTX_SUFF(pVM);
13258	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13259	TRPMResetTrap(pVCpu);
13260
13261	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13262	if ( rcStrict2 == VINF_SUCCESS
13263	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13264	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13265	{
13266	/* Successfully synced our nested page tables. */
13267	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13268	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13269	\| HM_CHANGED_GUEST_RSP
13270	\| HM_CHANGED_GUEST_RFLAGS);
13271	return VINF_SUCCESS;
13272	}
13273
13274	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13275	return rcStrict2;
13276	}
13277
13278	/** @} */
13279
13280	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13281	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13282	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13283
13284	/** @name VM-exit exception handlers.
13285	* @{
13286	*/
13287
13288	/**
13289	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13290	*/
13291	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13292	{
13293	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13294	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13295
13296	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13297	AssertRCReturn(rc, rc);
13298
13299	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13300	{
13301	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13302	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13303
13304	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13305	* provides VM-exit instruction length. If this causes problem later,
13306	* disassemble the instruction like it's done on AMD-V. */
13307	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13308	AssertRCReturn(rc2, rc2);
13309	return rc;
13310	}
13311
13312	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13313	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13314	return rc;
13315	}
13316
13317
13318	/**
13319	* VM-exit exception handler for \#BP (Breakpoint exception).
13320	*/
13321	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13322	{
13323	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13325
13326	/** @todo Try optimize this by not saving the entire guest state unless
13327	* really needed. */
13328	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13329	AssertRCReturn(rc, rc);
13330
13331	PVM pVM = pVCpu->CTX_SUFF(pVM);
13332	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13333	if (rc == VINF_EM_RAW_GUEST_TRAP)
13334	{
13335	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13336	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13337	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13338	AssertRCReturn(rc, rc);
13339
13340	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13341	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13342	}
13343
13344	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13345	return rc;
13346	}
13347
13348
13349	/**
13350	* VM-exit exception handler for \#AC (alignment check exception).
13351	*/
13352	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13353	{
13354	RT_NOREF_PV(pMixedCtx);
13355	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13356
13357	/*
13358	* Re-inject it. We'll detect any nesting before getting here.
13359	*/
13360	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13361	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13362	AssertRCReturn(rc, rc);
13363	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13364
13365	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13366	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13367	return VINF_SUCCESS;
13368	}
13369
13370
13371	/**
13372	* VM-exit exception handler for \#DB (Debug exception).
13373	*/
13374	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13375	{
13376	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13377	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13378	Log6(("XcptDB\n"));
13379
13380	/*
13381	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13382	* for processing.
13383	*/
13384	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13385	AssertRCReturn(rc, rc);
13386
13387	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13388	uint64_t uDR6 = X86_DR6_INIT_VAL;
13389	uDR6 \|= ( pVmxTransient->uExitQualification
13390	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13391
13392	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13393	if (rc == VINF_EM_RAW_GUEST_TRAP)
13394	{
13395	/*
13396	* The exception was for the guest. Update DR6, DR7.GD and
13397	* IA32_DEBUGCTL.LBR before forwarding it.
13398	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13399	*/
13400	VMMRZCallRing3Disable(pVCpu);
13401	HM_DISABLE_PREEMPT();
13402
13403	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13404	pMixedCtx->dr[6] \|= uDR6;
13405	if (CPUMIsGuestDebugStateActive(pVCpu))
13406	ASMSetDR6(pMixedCtx->dr[6]);
13407
13408	HM_RESTORE_PREEMPT();
13409	VMMRZCallRing3Enable(pVCpu);
13410
13411	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13412	AssertRCReturn(rc, rc);
13413
13414	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13415	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13416
13417	/* Paranoia. */
13418	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13419	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13420
13421	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13422	AssertRCReturn(rc, rc);
13423
13424	/*
13425	* Raise #DB in the guest.
13426	*
13427	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13428	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13429	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13430	*
13431	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13432	*/
13433	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13434	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13435	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13436	AssertRCReturn(rc, rc);
13437	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13438	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13439	return VINF_SUCCESS;
13440	}
13441
13442	/*
13443	* Not a guest trap, must be a hypervisor related debug event then.
13444	* Update DR6 in case someone is interested in it.
13445	*/
13446	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13447	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13448	CPUMSetHyperDR6(pVCpu, uDR6);
13449
13450	return rc;
13451	}
13452
13453
13454	/**
13455	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13456	* point exception).
13457	*/
13458	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13459	{
13460	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13461
13462	/* We require CR0 and EFER. EFER is always up-to-date. */
13463	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13464	AssertRCReturn(rc, rc);
13465
13466	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13467	VMMRZCallRing3Disable(pVCpu);
13468	HM_DISABLE_PREEMPT();
13469
13470	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13471	if (pVmxTransient->fWasGuestFPUStateActive)
13472	{
13473	rc = VINF_EM_RAW_GUEST_TRAP;
13474	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13475	}
13476	else
13477	{
13478	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13479	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13480	#endif
13481	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13482	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13483	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13484	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13485	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13486	}
13487
13488	HM_RESTORE_PREEMPT();
13489	VMMRZCallRing3Enable(pVCpu);
13490
13491	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13492	{
13493	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13494	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13496	pVCpu->hm.s.fPreloadGuestFpu = true;
13497	}
13498	else
13499	{
13500	/* Forward #NM to the guest. */
13501	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13502	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13503	AssertRCReturn(rc, rc);
13504	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13505	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13507	}
13508
13509	return VINF_SUCCESS;
13510	}
13511
13512
13513	/**
13514	* VM-exit exception handler for \#GP (General-protection exception).
13515	*
13516	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13517	*/
13518	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13519	{
13520	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13521	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13522
13523	int rc;
13524	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13525	{ /* likely */ }
13526	else
13527	{
13528	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13529	Assert(pVCpu->hm.s.fUsingDebugLoop);
13530	#endif
13531	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13532	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13533	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13534	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13535	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13536	AssertRCReturn(rc, rc);
13537	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13538	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13539	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13540	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13541	return rc;
13542	}
13543
13544	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13545	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13546
13547	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13548	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13549	AssertRCReturn(rc, rc);
13550
13551	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13552	uint32_t cbOp = 0;
13553	PVM pVM = pVCpu->CTX_SUFF(pVM);
13554	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13555	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13556	if (RT_SUCCESS(rc))
13557	{
13558	rc = VINF_SUCCESS;
13559	Assert(cbOp == pDis->cbInstr);
13560	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13561	switch (pDis->pCurInstr->uOpcode)
13562	{
13563	case OP_CLI:
13564	{
13565	pMixedCtx->eflags.Bits.u1IF = 0;
13566	pMixedCtx->eflags.Bits.u1RF = 0;
13567	pMixedCtx->rip += pDis->cbInstr;
13568	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13569	if ( !fDbgStepping
13570	&& pMixedCtx->eflags.Bits.u1TF)
13571	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13572	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13573	break;
13574	}
13575
13576	case OP_STI:
13577	{
13578	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13579	pMixedCtx->eflags.Bits.u1IF = 1;
13580	pMixedCtx->eflags.Bits.u1RF = 0;
13581	pMixedCtx->rip += pDis->cbInstr;
13582	if (!fOldIF)
13583	{
13584	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13585	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13586	}
13587	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13588	if ( !fDbgStepping
13589	&& pMixedCtx->eflags.Bits.u1TF)
13590	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13592	break;
13593	}
13594
13595	case OP_HLT:
13596	{
13597	rc = VINF_EM_HALT;
13598	pMixedCtx->rip += pDis->cbInstr;
13599	pMixedCtx->eflags.Bits.u1RF = 0;
13600	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13602	break;
13603	}
13604
13605	case OP_POPF:
13606	{
13607	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13608	uint32_t cbParm;
13609	uint32_t uMask;
13610	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13611	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13612	{
13613	cbParm = 4;
13614	uMask = 0xffffffff;
13615	}
13616	else
13617	{
13618	cbParm = 2;
13619	uMask = 0xffff;
13620	}
13621
13622	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13623	RTGCPTR GCPtrStack = 0;
13624	X86EFLAGS Eflags;
13625	Eflags.u32 = 0;
13626	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13627	&GCPtrStack);
13628	if (RT_SUCCESS(rc))
13629	{
13630	Assert(sizeof(Eflags.u32) >= cbParm);
13631	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13632	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13633	}
13634	if (RT_FAILURE(rc))
13635	{
13636	rc = VERR_EM_INTERPRETER;
13637	break;
13638	}
13639	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13640	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13641	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13642	pMixedCtx->esp += cbParm;
13643	pMixedCtx->esp &= uMask;
13644	pMixedCtx->rip += pDis->cbInstr;
13645	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13646	\| HM_CHANGED_GUEST_RSP
13647	\| HM_CHANGED_GUEST_RFLAGS);
13648	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13649	POPF restores EFLAGS.TF. */
13650	if ( !fDbgStepping
13651	&& fGstStepping)
13652	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13653	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13654	break;
13655	}
13656
13657	case OP_PUSHF:
13658	{
13659	uint32_t cbParm;
13660	uint32_t uMask;
13661	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13662	{
13663	cbParm = 4;
13664	uMask = 0xffffffff;
13665	}
13666	else
13667	{
13668	cbParm = 2;
13669	uMask = 0xffff;
13670	}
13671
13672	/* Get the stack pointer & push the contents of eflags onto the stack. */
13673	RTGCPTR GCPtrStack = 0;
13674	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13675	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13676	if (RT_FAILURE(rc))
13677	{
13678	rc = VERR_EM_INTERPRETER;
13679	break;
13680	}
13681	X86EFLAGS Eflags = pMixedCtx->eflags;
13682	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13683	Eflags.Bits.u1RF = 0;
13684	Eflags.Bits.u1VM = 0;
13685
13686	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13687	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13688	{
13689	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13690	rc = VERR_EM_INTERPRETER;
13691	break;
13692	}
13693	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13694	pMixedCtx->esp -= cbParm;
13695	pMixedCtx->esp &= uMask;
13696	pMixedCtx->rip += pDis->cbInstr;
13697	pMixedCtx->eflags.Bits.u1RF = 0;
13698	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13699	\| HM_CHANGED_GUEST_RSP
13700	\| HM_CHANGED_GUEST_RFLAGS);
13701	if ( !fDbgStepping
13702	&& pMixedCtx->eflags.Bits.u1TF)
13703	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13704	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13705	break;
13706	}
13707
13708	case OP_IRET:
13709	{
13710	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13711	* instruction reference. */
13712	RTGCPTR GCPtrStack = 0;
13713	uint32_t uMask = 0xffff;
13714	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13715	uint16_t aIretFrame[3];
13716	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13717	{
13718	rc = VERR_EM_INTERPRETER;
13719	break;
13720	}
13721	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13722	&GCPtrStack);
13723	if (RT_SUCCESS(rc))
13724	{
13725	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13726	PGMACCESSORIGIN_HM));
13727	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13728	}
13729	if (RT_FAILURE(rc))
13730	{
13731	rc = VERR_EM_INTERPRETER;
13732	break;
13733	}
13734	pMixedCtx->eip = 0;
13735	pMixedCtx->ip = aIretFrame[0];
13736	pMixedCtx->cs.Sel = aIretFrame[1];
13737	pMixedCtx->cs.ValidSel = aIretFrame[1];
13738	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13739	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13740	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13741	pMixedCtx->sp += sizeof(aIretFrame);
13742	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13743	\| HM_CHANGED_GUEST_SEGMENT_REGS
13744	\| HM_CHANGED_GUEST_RSP
13745	\| HM_CHANGED_GUEST_RFLAGS);
13746	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13747	if ( !fDbgStepping
13748	&& fGstStepping)
13749	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13750	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13751	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13752	break;
13753	}
13754
13755	case OP_INT:
13756	{
13757	uint16_t uVector = pDis->Param1.uValue & 0xff;
13758	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13759	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13760	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13761	break;
13762	}
13763
13764	case OP_INTO:
13765	{
13766	if (pMixedCtx->eflags.Bits.u1OF)
13767	{
13768	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13769	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13770	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13771	}
13772	else
13773	{
13774	pMixedCtx->eflags.Bits.u1RF = 0;
13775	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13776	}
13777	break;
13778	}
13779
13780	default:
13781	{
13782	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13783	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13784	EMCODETYPE_SUPERVISOR);
13785	rc = VBOXSTRICTRC_VAL(rc2);
13786	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13787	/** @todo We have to set pending-debug exceptions here when the guest is
13788	* single-stepping depending on the instruction that was interpreted. */
13789	Log4(("#GP rc=%Rrc\n", rc));
13790	break;
13791	}
13792	}
13793	}
13794	else
13795	rc = VERR_EM_INTERPRETER;
13796
13797	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13798	("#GP Unexpected rc=%Rrc\n", rc));
13799	return rc;
13800	}
13801
13802
13803	/**
13804	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13805	* the exception reported in the VMX transient structure back into the VM.
13806	*
13807	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13808	* up-to-date.
13809	*/
13810	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13811	{
13812	RT_NOREF_PV(pMixedCtx);
13813	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13814	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13815	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13816	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13817	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13818	#endif
13819
13820	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13821	hmR0VmxCheckExitDueToEventDelivery(). */
13822	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13823	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13824	AssertRCReturn(rc, rc);
13825	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13826
13827	#ifdef DEBUG_ramshankar
13828	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13829	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13830	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13831	#endif
13832
13833	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13834	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13835	return VINF_SUCCESS;
13836	}
13837
13838
13839	/**
13840	* VM-exit exception handler for \#PF (Page-fault exception).
13841	*/
13842	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13843	{
13844	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13845	PVM pVM = pVCpu->CTX_SUFF(pVM);
13846	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13847	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13848	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13849	AssertRCReturn(rc, rc);
13850
13851	if (!pVM->hm.s.fNestedPaging)
13852	{ /* likely */ }
13853	else
13854	{
13855	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13856	Assert(pVCpu->hm.s.fUsingDebugLoop);
13857	#endif
13858	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13859	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13860	{
13861	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13862	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13863	}
13864	else
13865	{
13866	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13867	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13868	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13869	}
13870	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13871	return rc;
13872	}
13873
13874	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13875	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13876	if (pVmxTransient->fVectoringPF)
13877	{
13878	Assert(pVCpu->hm.s.Event.fPending);
13879	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13880	}
13881
13882	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13883	AssertRCReturn(rc, rc);
13884
13885	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13886	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13887
13888	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13889	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13890	(RTGCPTR)pVmxTransient->uExitQualification);
13891
13892	Log4(("#PF: rc=%Rrc\n", rc));
13893	if (rc == VINF_SUCCESS)
13894	{
13895	#if 0
13896	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13897	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13898	* memory? We don't update the whole state here... */
13899	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13900	\| HM_CHANGED_GUEST_RSP
13901	\| HM_CHANGED_GUEST_RFLAGS
13902	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13903	#else
13904	/*
13905	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13906	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13907	*/
13908	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13909	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13910	#endif
13911	TRPMResetTrap(pVCpu);
13912	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13913	return rc;
13914	}
13915
13916	if (rc == VINF_EM_RAW_GUEST_TRAP)
13917	{
13918	if (!pVmxTransient->fVectoringDoublePF)
13919	{
13920	/* It's a guest page fault and needs to be reflected to the guest. */
13921	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13922	TRPMResetTrap(pVCpu);
13923	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13924	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13925	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13926	}
13927	else
13928	{
13929	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13930	TRPMResetTrap(pVCpu);
13931	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13932	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13933	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13934	}
13935
13936	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13937	return VINF_SUCCESS;
13938	}
13939
13940	TRPMResetTrap(pVCpu);
13941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13942	return rc;
13943	}
13944
13945	/** @} */
13946

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 67116

Download in other formats: